Initial snapshot before transformerlab recovery
This commit is contained in:
Codex
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create or refresh a default local TransformerLab user."
|
||||
)
|
||||
parser.add_argument("--transformerlab-dir", required=True, help="Path to the TransformerLab home directory")
|
||||
parser.add_argument("--email", required=True, help="Email address for the default user")
|
||||
parser.add_argument("--password", required=True, help="Password for the default user")
|
||||
parser.add_argument("--first-name", default="", help="Optional first name")
|
||||
parser.add_argument("--last-name", default="", help="Optional last name")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_environment(transformerlab_dir: Path) -> None:
|
||||
env_file = transformerlab_dir / ".env"
|
||||
if env_file.exists():
|
||||
try:
|
||||
from dotenv import load_dotenv
|
||||
except ImportError:
|
||||
return
|
||||
load_dotenv(env_file)
|
||||
|
||||
|
||||
async def ensure_user(
|
||||
transformerlab_dir: Path,
|
||||
email: str,
|
||||
password: str,
|
||||
first_name: str,
|
||||
last_name: str,
|
||||
) -> None:
|
||||
src_dir = transformerlab_dir / "src"
|
||||
if not src_dir.exists():
|
||||
raise FileNotFoundError(f"TransformerLab source directory not found: {src_dir}")
|
||||
|
||||
sys.path.insert(0, str(src_dir))
|
||||
load_environment(transformerlab_dir)
|
||||
|
||||
from fastapi_users.db import SQLAlchemyUserDatabase
|
||||
from sqlalchemy import select
|
||||
from transformerlab.models.users import UserCreate, UserManager, UserUpdate
|
||||
from transformerlab.services.provider_service import initialize_team_local_provider
|
||||
from transformerlab.shared.models.models import TeamRole, User, UserTeam
|
||||
from transformerlab.shared.models.user_model import AsyncSessionLocal, create_personal_team
|
||||
|
||||
async with AsyncSessionLocal() as session:
|
||||
user_db = SQLAlchemyUserDatabase(session, User)
|
||||
user_manager = UserManager(user_db)
|
||||
|
||||
stmt = select(User).where(User.email == email)
|
||||
result = await session.execute(stmt)
|
||||
existing_user = result.unique().scalar_one_or_none()
|
||||
|
||||
created = existing_user is None
|
||||
if created:
|
||||
await user_manager.create(
|
||||
UserCreate(
|
||||
email=email,
|
||||
password=password,
|
||||
is_active=True,
|
||||
is_superuser=False,
|
||||
is_verified=True,
|
||||
first_name=first_name or None,
|
||||
last_name=last_name or None,
|
||||
),
|
||||
safe=False,
|
||||
request=None,
|
||||
)
|
||||
else:
|
||||
await user_manager.update(
|
||||
UserUpdate(
|
||||
password=password,
|
||||
is_active=True,
|
||||
is_superuser=False,
|
||||
is_verified=True,
|
||||
first_name=first_name or existing_user.first_name,
|
||||
last_name=last_name or existing_user.last_name,
|
||||
),
|
||||
existing_user,
|
||||
safe=False,
|
||||
request=None,
|
||||
)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
user = result.unique().scalar_one()
|
||||
user_id = str(user.id)
|
||||
|
||||
team_stmt = select(UserTeam).where(UserTeam.user_id == user_id).limit(1)
|
||||
team_result = await session.execute(team_stmt)
|
||||
user_team = team_result.scalar_one_or_none()
|
||||
|
||||
if user_team is None:
|
||||
personal_team = await create_personal_team(session, user)
|
||||
user_team = UserTeam(user_id=user_id, team_id=personal_team.id, role=TeamRole.OWNER.value)
|
||||
session.add(user_team)
|
||||
await session.commit()
|
||||
team_id = personal_team.id
|
||||
else:
|
||||
team_id = user_team.team_id
|
||||
|
||||
try:
|
||||
await initialize_team_local_provider(session, team_id, user_id)
|
||||
except Exception as exc:
|
||||
print(f"warning: failed to initialize local provider for {email}: {exc}")
|
||||
|
||||
print(
|
||||
f"{'created' if created else 'updated'} default TransformerLab user {email} "
|
||||
f"(team_id={team_id})"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
transformerlab_dir = Path(args.transformerlab_dir).expanduser().resolve()
|
||||
|
||||
try:
|
||||
asyncio.run(
|
||||
ensure_user(
|
||||
transformerlab_dir=transformerlab_dir,
|
||||
email=args.email,
|
||||
password=args.password,
|
||||
first_name=args.first_name,
|
||||
last_name=args.last_name,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
STATE_DIR="$ROOT_DIR/state"
|
||||
RUNTIME_ENV="$STATE_DIR/runtime.env"
|
||||
|
||||
load_runtime_env() {
|
||||
if [ -f "$RUNTIME_ENV" ]; then
|
||||
# shellcheck disable=SC1090
|
||||
. "$RUNTIME_ENV"
|
||||
fi
|
||||
|
||||
: "${COURSEWARE_STATE_DIR:=$STATE_DIR}"
|
||||
: "${COURSEWARE_BIND_HOST:=127.0.0.1}"
|
||||
: "${COURSEWARE_URL_HOST:=127.0.0.1}"
|
||||
: "${COURSEWARE_PROMPTFOO_PORT:=15500}"
|
||||
: "${COURSEWARE_WIKI_PORT:=80}"
|
||||
: "${NODE_RUNTIME_BIN_DIR:=$COURSEWARE_STATE_DIR/tools/node-runtime/node_modules/node/bin}"
|
||||
: "${PROMPTFOO_DIR:=$COURSEWARE_STATE_DIR/lab6}"
|
||||
: "${WIKI_DIR:=$COURSEWARE_STATE_DIR/repos/LLM-Labs}"
|
||||
: "${LLAMA_CPP_BIN_DIR:=$COURSEWARE_STATE_DIR/repos/llama.cpp/build/bin}"
|
||||
}
|
||||
|
||||
ensure_runtime_env() {
|
||||
if [ ! -f "$RUNTIME_ENV" ]; then
|
||||
echo "Missing $RUNTIME_ENV. Run ./labctl up first." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
service_list() {
|
||||
printf '%s\n' \
|
||||
"ollama" \
|
||||
"open-webui" \
|
||||
"transformerlab" \
|
||||
"chunkviz" \
|
||||
"embedding-atlas" \
|
||||
"unsloth" \
|
||||
"promptfoo" \
|
||||
"wiki"
|
||||
}
|
||||
|
||||
service_pid_file() {
|
||||
printf '%s/run/%s.pid\n' "$STATE_DIR" "${1//-/_}"
|
||||
}
|
||||
|
||||
service_log_file() {
|
||||
printf '%s/logs/%s.log\n' "$STATE_DIR" "${1//-/_}"
|
||||
}
|
||||
|
||||
service_port() {
|
||||
case "$1" in
|
||||
ollama) printf '%s\n' "${COURSEWARE_OLLAMA_PORT}" ;;
|
||||
open-webui) printf '%s\n' "${COURSEWARE_OPEN_WEBUI_PORT}" ;;
|
||||
transformerlab) printf '%s\n' "${COURSEWARE_TRANSFORMERLAB_PORT}" ;;
|
||||
chunkviz) printf '%s\n' "${COURSEWARE_CHUNKVIZ_PORT}" ;;
|
||||
embedding-atlas) printf '%s\n' "${COURSEWARE_EMBEDDING_ATLAS_PORT}" ;;
|
||||
unsloth) printf '%s\n' "${COURSEWARE_UNSLOTH_PORT}" ;;
|
||||
promptfoo) printf '%s\n' "${COURSEWARE_PROMPTFOO_PORT}" ;;
|
||||
wiki) printf '%s\n' "${COURSEWARE_WIKI_PORT}" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
service_url() {
|
||||
case "$1" in
|
||||
ollama) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_OLLAMA_PORT" ;;
|
||||
open-webui) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_OPEN_WEBUI_PORT" ;;
|
||||
transformerlab) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_TRANSFORMERLAB_PORT" ;;
|
||||
chunkviz) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_CHUNKVIZ_PORT" ;;
|
||||
embedding-atlas) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_EMBEDDING_ATLAS_PORT" ;;
|
||||
unsloth) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_UNSLOTH_PORT" ;;
|
||||
promptfoo) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_PROMPTFOO_PORT" ;;
|
||||
wiki) printf 'http://%s:%s\n' "$COURSEWARE_URL_HOST" "$COURSEWARE_WIKI_PORT" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
service_command() {
|
||||
case "$1" in
|
||||
ollama)
|
||||
printf 'exec env OLLAMA_HOST=%s:%s OLLAMA_MODELS="%s" "%s" serve' \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_OLLAMA_PORT" \
|
||||
"$OLLAMA_MODELS_DIR" \
|
||||
"$OLLAMA_BIN"
|
||||
;;
|
||||
open-webui)
|
||||
printf 'exec env DATA_DIR="%s" OLLAMA_BASE_URL=http://%s:%s "%s/bin/open-webui" serve --host %s --port %s' \
|
||||
"$OPEN_WEBUI_DATA_DIR" \
|
||||
"$COURSEWARE_URL_HOST" \
|
||||
"$COURSEWARE_OLLAMA_PORT" \
|
||||
"$OPEN_WEBUI_VENV" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_OPEN_WEBUI_PORT"
|
||||
;;
|
||||
transformerlab)
|
||||
printf 'cd "%s/src" && exec ./run.sh -h %s -p %s' \
|
||||
"$TRANSFORMERLAB_DIR" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_TRANSFORMERLAB_PORT"
|
||||
;;
|
||||
chunkviz)
|
||||
printf 'cd "%s" && PATH="%s:$PATH" exec "./node_modules/.bin/serve" build -s -n -L -l tcp://%s:%s' \
|
||||
"$CHUNKVIZ_DIR" \
|
||||
"$NODE_RUNTIME_BIN_DIR" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_CHUNKVIZ_PORT"
|
||||
;;
|
||||
embedding-atlas)
|
||||
printf 'exec "%s/bin/embedding-atlas" "%s" --text "Scenario" --host %s --port %s' \
|
||||
"$EMBEDDING_ATLAS_VENV" \
|
||||
"$TTPS_DATASET_PATH" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_EMBEDDING_ATLAS_PORT"
|
||||
;;
|
||||
unsloth)
|
||||
printf 'exec "%s" studio -H %s -p %s' \
|
||||
"$UNSLOTH_BIN" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_UNSLOTH_PORT"
|
||||
;;
|
||||
promptfoo)
|
||||
printf 'cd "%s" && PATH="%s:$PATH" COURSEWARE_BIND_HOST=%s "%s" view . --port %s --no' \
|
||||
"$PROMPTFOO_DIR" \
|
||||
"$NODE_RUNTIME_BIN_DIR" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$PROMPTFOO_BIN" \
|
||||
"$COURSEWARE_PROMPTFOO_PORT"
|
||||
;;
|
||||
wiki)
|
||||
printf 'cd "%s" && PATH="%s:$PATH" exec "./node_modules/.bin/next" start --hostname %s --port %s' \
|
||||
"$WIKI_DIR" \
|
||||
"$NODE_RUNTIME_BIN_DIR" \
|
||||
"$COURSEWARE_BIND_HOST" \
|
||||
"$COURSEWARE_WIKI_PORT"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck disable=SC1091
|
||||
. "$SCRIPT_DIR/common.sh"
|
||||
|
||||
load_runtime_env
|
||||
|
||||
mkdir -p "$STATE_DIR/run" "$STATE_DIR/logs"
|
||||
|
||||
resolve_targets() {
|
||||
if [ $# -eq 0 ]; then
|
||||
echo "No target specified." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$1" in
|
||||
core)
|
||||
printf '%s\n' "ollama" "open-webui"
|
||||
;;
|
||||
all)
|
||||
service_list
|
||||
;;
|
||||
*)
|
||||
printf '%s\n' "$@"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
has_live_pid() {
|
||||
local service=$1
|
||||
local pid_file
|
||||
pid_file=$(service_pid_file "$service")
|
||||
|
||||
if [ -f "$pid_file" ]; then
|
||||
local pid
|
||||
pid=$(cat "$pid_file")
|
||||
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
is_running() {
|
||||
local service=$1
|
||||
|
||||
has_live_pid "$service" || service_ready "$service"
|
||||
}
|
||||
|
||||
service_ready() {
|
||||
local service=$1
|
||||
|
||||
case "$service" in
|
||||
ollama)
|
||||
curl -fsS "$(service_url "$service")/api/tags" >/dev/null 2>&1
|
||||
;;
|
||||
promptfoo)
|
||||
curl -fsS "$(service_url "$service")/health" >/dev/null 2>&1
|
||||
;;
|
||||
open-webui|transformerlab|chunkviz|embedding-atlas|unsloth|wiki)
|
||||
curl -fsS "$(service_url "$service")" >/dev/null 2>&1
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
start_one() {
|
||||
local service=$1
|
||||
local cmd
|
||||
local log_file
|
||||
local pid_file
|
||||
local attempt
|
||||
|
||||
if has_live_pid "$service"; then
|
||||
echo "$service already running"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if service_ready "$service"; then
|
||||
echo "$service already available"
|
||||
return 0
|
||||
fi
|
||||
|
||||
case "$service" in
|
||||
open-webui)
|
||||
start_one ollama
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
|
||||
cmd=$(service_command "$service")
|
||||
log_file=$(service_log_file "$service")
|
||||
pid_file=$(service_pid_file "$service")
|
||||
|
||||
if command -v setsid >/dev/null 2>&1; then
|
||||
nohup setsid bash -lc "$cmd" </dev/null >>"$log_file" 2>&1 &
|
||||
else
|
||||
nohup bash -lc "$cmd" </dev/null >>"$log_file" 2>&1 &
|
||||
fi
|
||||
echo $! >"$pid_file"
|
||||
|
||||
for attempt in $(seq 1 60); do
|
||||
if service_ready "$service"; then
|
||||
echo "started $service"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! has_live_pid "$service"; then
|
||||
rm -f "$pid_file"
|
||||
echo "failed to start $service; check $log_file" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "$service did not become ready in time; check $log_file" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
stop_one() {
|
||||
local service=$1
|
||||
local pid_file
|
||||
|
||||
pid_file=$(service_pid_file "$service")
|
||||
|
||||
if [ ! -f "$pid_file" ]; then
|
||||
echo "$service not running"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local pid
|
||||
pid=$(cat "$pid_file")
|
||||
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||
kill "$pid" >/dev/null 2>&1 || true
|
||||
sleep 2
|
||||
if kill -0 "$pid" >/dev/null 2>&1; then
|
||||
kill -9 "$pid" >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$pid_file"
|
||||
echo "stopped $service"
|
||||
}
|
||||
|
||||
status_one() {
|
||||
local service=$1
|
||||
|
||||
if service_ready "$service"; then
|
||||
printf 'RUNNING %-15s %s\n' "$service" "$(service_url "$service")"
|
||||
elif has_live_pid "$service"; then
|
||||
printf 'STARTING %-15s %s\n' "$service" "$(service_url "$service")"
|
||||
else
|
||||
printf 'STOPPED %-15s %s\n' "$service" "$(service_url "$service")"
|
||||
fi
|
||||
}
|
||||
|
||||
urls() {
|
||||
cat <<EOF
|
||||
Ollama API: $(service_url ollama)
|
||||
Open WebUI: $(service_url open-webui)
|
||||
TransformerLab: $(service_url transformerlab)
|
||||
ChunkViz: $(service_url chunkviz)
|
||||
Embedding Atlas: $(service_url embedding-atlas)
|
||||
Unsloth Studio: $(service_url unsloth)
|
||||
Promptfoo CLI: $PROMPTFOO_BIN
|
||||
Promptfoo UI: $(service_url promptfoo)
|
||||
Wiki: $(service_url wiki)
|
||||
Kiln app: ${KILN_LAUNCH_PATH:-not installed}
|
||||
EOF
|
||||
}
|
||||
|
||||
open_kiln() {
|
||||
local host_os
|
||||
|
||||
host_os=$(uname -s)
|
||||
if [ "$host_os" = "Darwin" ] && [ -d "$KILN_MAC_APP" ]; then
|
||||
open "$KILN_MAC_APP"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -x "$KILN_LINUX_BIN" ]; then
|
||||
nohup "$KILN_LINUX_BIN" >/dev/null 2>&1 &
|
||||
echo "started Kiln from $KILN_LINUX_BIN"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Kiln is not installed." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
show_logs() {
|
||||
local service=$1
|
||||
local log_file
|
||||
|
||||
log_file=$(service_log_file "$service")
|
||||
if [ ! -f "$log_file" ]; then
|
||||
echo "No log file for $service" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tail -n 80 "$log_file"
|
||||
}
|
||||
|
||||
main() {
|
||||
local cmd=${1:-}
|
||||
shift || true
|
||||
|
||||
ensure_runtime_env
|
||||
|
||||
case "$cmd" in
|
||||
start)
|
||||
while IFS= read -r service; do
|
||||
start_one "$service"
|
||||
done < <(resolve_targets "$@")
|
||||
;;
|
||||
stop)
|
||||
while IFS= read -r service; do
|
||||
stop_one "$service"
|
||||
done < <(resolve_targets "$@")
|
||||
;;
|
||||
status)
|
||||
if [ $# -eq 0 ]; then
|
||||
set -- all
|
||||
fi
|
||||
while IFS= read -r service; do
|
||||
status_one "$service"
|
||||
done < <(resolve_targets "$@")
|
||||
;;
|
||||
urls)
|
||||
urls
|
||||
;;
|
||||
open)
|
||||
if [ "${1:-}" != "kiln" ]; then
|
||||
echo "Only 'open kiln' is supported." >&2
|
||||
exit 1
|
||||
fi
|
||||
open_kiln
|
||||
;;
|
||||
logs)
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: ./labctl logs <service>" >&2
|
||||
exit 1
|
||||
fi
|
||||
show_logs "$1"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown command: $cmd" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user