#!/usr/bin/env bash
set -eu
set -o errexit
set -o nounset
set -o pipefail
readonly SCRIPT_SRC="$(dirname "${BASH_SOURCE[${#BASH_SOURCE[@]} - 1]}")"
readonly SCRIPT_DIR="$(cd "${SCRIPT_SRC}" >/dev/null 2>&1 && pwd)"
readonly SCRIPT_NAME=$(basename "$0")
# Avoid issues when wine is installed.
sudo su -c 'echo 0 > /proc/sys/fs/binfmt_misc/status'
# Graceful exit to perform any clean up, if needed.
trap terminate INT
# Exits the script with a given error level.
function terminate() {
level=10
if [ $# -ge 1 ] && [ -n "$1" ]; then level="$1"; fi
exit $level
}
# Concatenates multiple files.
join() {
local -r prefix="$1"
local -r content="$2"
local -r suffix="$3"
printf "%s%s%s" "$(cat ${prefix})" "$(cat ${content})" "$(cat ${suffix})"
}
# Swapping this symbolic link allows swapping the LLM without script changes.
readonly LINK_MODEL="${SCRIPT_DIR}/llm.gguf"
# Dereference the model's symbolic link to its path relative to the script.
readonly PATH_MODEL="$(realpath --relative-to="${SCRIPT_DIR}" "${LINK_MODEL}")"
# Extract the file name for the model.
readonly FILE_MODEL=$(basename "${PATH_MODEL}")
# Look up the prompt format based on the model being used.
readonly PROMPT_FORMAT=$(grep -m1 ${FILE_MODEL} map.txt | sed 's/.*: //')
# Guard against missing prompt templates.
if [ -z "${PROMPT_FORMAT}" ]; then
echo "Add prompt template for '${FILE_MODEL}'."
terminate 11
fi
readonly FILE_MODEL_NAME=$(basename $FILE_MODEL)
if [ -z "${1:-}" ]; then
# Write the output to a name corresponding to the model being used.
PATH_OUTPUT="output/${FILE_MODEL_NAME%.*}.txt"
else
PATH_OUTPUT="$1"
fi
# The system file defines the parameters of the interaction.
readonly PATH_PROMPT_SYSTEM="system.txt"
# The user file prompts the model as to what we want to generate.
readonly PATH_PROMPT_USER="user.txt"
readonly PATH_PREFIX_SYSTEM="templates/${PROMPT_FORMAT}/prefix-system.txt"
readonly PATH_PREFIX_USER="templates/${PROMPT_FORMAT}/prefix-user.txt"
readonly PATH_PREFIX_ASSIST="templates/${PROMPT_FORMAT}/prefix-assistant.txt"
readonly PATH_SUFFIX_SYSTEM="templates/${PROMPT_FORMAT}/suffix-system.txt"
readonly PATH_SUFFIX_USER="templates/${PROMPT_FORMAT}/suffix-user.txt"
readonly PATH_SUFFIX_ASSIST="templates/${PROMPT_FORMAT}/suffix-assistant.txt"
echo "Running: ${PATH_MODEL}"
echo "Reading: ${PATH_PREFIX_SYSTEM}"
echo "Reading: ${PATH_PREFIX_USER}"
echo "Reading: ${PATH_PREFIX_ASSIST}"
echo "Writing: ${PATH_OUTPUT}"
# Capture the entirety of the instructions to obtain the input length.
readonly INSTRUCT=$(
join ${PATH_PREFIX_SYSTEM} ${PATH_PROMPT_SYSTEM} ${PATH_PREFIX_SYSTEM}
join ${PATH_SUFFIX_USER} ${PATH_PROMPT_USER} ${PATH_SUFFIX_USER}
join ${PATH_SUFFIX_ASSIST} "/dev/null" ${PATH_SUFFIX_ASSIST}
)
(
echo ${INSTRUCT}
) | ./llamafile \
-m "${LINK_MODEL}" \
-e \
-f /dev/stdin \
-n 1000 \
-c ${#INSTRUCT} \
--repeat-penalty 1.0 \
--temp 0.3 \
--silent-prompt > ${PATH_OUTPUT}
#--log-disable \
echo "Outputs: ${PATH_OUTPUT}"
terminate 0
