From 54ff24165ebe7e5e93f5bab98d556b4c9d56eae2 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 17:03:36 -0500 Subject: [PATCH 01/20] fix alembic migration --- fileglancer/alembic.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fileglancer/alembic.ini b/fileglancer/alembic.ini index 2b2d7ac5..2583713a 100644 --- a/fileglancer/alembic.ini +++ b/fileglancer/alembic.ini @@ -2,7 +2,7 @@ [alembic] # path to migration scripts -script_location = alembic +script_location = fileglancer/alembic # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s # Uncomment the line below if you want the files to be prepended with date and time From e8f0f82ed1f7eb759bee17ec7f16c95a28bbf634 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 17:04:49 -0500 Subject: [PATCH 02/20] added service apps --- ...e8a3f21c76_add_entry_point_type_to_jobs.py | 24 ++++++ fileglancer/apps.py | 41 ++++++++++- fileglancer/database.py | 3 + fileglancer/model.py | 3 + fileglancer/server.py | 2 + frontend/src/components/AppLaunch.tsx | 22 ++++-- frontend/src/components/JobDetail.tsx | 73 ++++++++++++++++++- .../components/ui/AppsPage/AppLaunchForm.tsx | 4 +- .../components/ui/Table/appsJobsColumns.tsx | 21 +++++- frontend/src/shared.types.ts | 3 + pixi.lock | 2 +- pyproject.toml | 4 +- 12 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 fileglancer/alembic/versions/b5e8a3f21c76_add_entry_point_type_to_jobs.py diff --git a/fileglancer/alembic/versions/b5e8a3f21c76_add_entry_point_type_to_jobs.py b/fileglancer/alembic/versions/b5e8a3f21c76_add_entry_point_type_to_jobs.py new file mode 100644 index 00000000..c8f529fc --- /dev/null +++ b/fileglancer/alembic/versions/b5e8a3f21c76_add_entry_point_type_to_jobs.py @@ -0,0 +1,24 @@ +"""add entry_point_type to jobs + +Revision ID: b5e8a3f21c76 +Revises: a3f7c2e19d04 +Create Date: 2026-02-23 00:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b5e8a3f21c76' +down_revision = 'a3f7c2e19d04' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column('jobs', sa.Column('entry_point_type', sa.String(), nullable=False, server_default='job')) + + +def downgrade() -> None: + op.drop_column('jobs', 'entry_point_type') diff --git a/fileglancer/apps.py b/fileglancer/apps.py index cf05a9be..63c2818c 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -627,6 +627,7 @@ async def submit_job( app_name=manifest.name, entry_point_id=entry_point.id, entry_point_name=entry_point.name, + entry_point_type=entry_point.type, parameters=parameters, resources=resources_dict, manifest_path=manifest_path, @@ -798,8 +799,37 @@ def _make_file_info(file_path: str, exists: bool) -> dict: } +def get_service_url(db_job: db.JobDB) -> Optional[str]: + """Read the service URL from a job's work directory. + + Only returns a URL when the job is a service type and is currently RUNNING. + The service writes its URL to a plain text file named 'service_url' in the + job's work directory. + """ + if getattr(db_job, 'entry_point_type', 'job') != 'service': + return None + if db_job.status != 'RUNNING': + return None + + work_dir = _resolve_work_dir(db_job) + url_file = work_dir / "service_url" + + if not url_file.is_file(): + return None + + try: + url = url_file.read_text().strip() + except OSError: + return None + + if not url.startswith(("http://", "https://")): + return None + + return url + + def get_job_file_paths(db_job: db.JobDB) -> dict[str, dict]: - """Return file path info for a job's files (script, stdout, stderr). + """Return file path info for a job's files (script, stdout, stderr, service_url). Returns a dict keyed by file type with path and existence info. """ @@ -812,12 +842,19 @@ def get_job_file_paths(db_job: db.JobDB) -> dict[str, dict]: stdout_path = work_dir / "stdout.log" stderr_path = work_dir / "stderr.log" - return { + files = { "script": _make_file_info(script_path, len(scripts) > 0), "stdout": _make_file_info(str(stdout_path), stdout_path.is_file()), "stderr": _make_file_info(str(stderr_path), stderr_path.is_file()), } + # Include service_url file info for service-type jobs + if getattr(db_job, 'entry_point_type', 'job') == 'service': + service_url_path = work_dir / "service_url" + files["service_url"] = _make_file_info(str(service_url_path), service_url_path.is_file()) + + return files + async def get_job_file_content(job_id: int, username: str, file_type: str) -> Optional[str]: """Read the content of a job file (script, stdout, or stderr). diff --git a/fileglancer/database.py b/fileglancer/database.py index 5feb0c92..7adfdb63 100644 --- a/fileglancer/database.py +++ b/fileglancer/database.py @@ -148,6 +148,7 @@ class JobDB(Base): manifest_path = Column(String, nullable=False, server_default="") entry_point_id = Column(String, nullable=False) entry_point_name = Column(String, nullable=False) + entry_point_type = Column(String, nullable=False, server_default="job") parameters = Column(JSON, nullable=False) status = Column(String, nullable=False, default="PENDING") exit_code = Column(Integer, nullable=True) @@ -828,6 +829,7 @@ def delete_expired_sessions(session: Session): def create_job(session: Session, username: str, app_url: str, app_name: str, entry_point_id: str, entry_point_name: str, parameters: Dict, resources: Optional[Dict] = None, manifest_path: str = "", + entry_point_type: str = "job", env: Optional[Dict] = None, pre_run: Optional[str] = None, post_run: Optional[str] = None, pull_latest: bool = False) -> JobDB: """Create a new job record""" @@ -839,6 +841,7 @@ def create_job(session: Session, username: str, app_url: str, app_name: str, manifest_path=manifest_path, entry_point_id=entry_point_id, entry_point_name=entry_point_name, + entry_point_type=entry_point_type, parameters=parameters, resources=resources, env=env, diff --git a/fileglancer/model.py b/fileglancer/model.py index 7cf897a9..45fde4b5 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -382,6 +382,7 @@ class AppEntryPoint(BaseModel): """An entry point (command) within an app""" id: str = Field(description="Unique identifier for the entry point") name: str = Field(description="Display name of the entry point") + type: Literal["job", "service"] = Field(description="Whether this is a batch job or long-running service", default="job") description: Optional[str] = Field(description="Description of the entry point", default=None) command: str = Field(description="The base CLI command to execute") parameters: List[AppParameterItem] = Field(description="Parameters for this entry point", default=[]) @@ -489,6 +490,7 @@ class Job(BaseModel): manifest_path: str = Field(description="Relative manifest path within the app repo", default="") entry_point_id: str = Field(description="Entry point that was executed") entry_point_name: str = Field(description="Display name of the entry point") + entry_point_type: str = Field(description="Whether this is a batch job or long-running service", default="job") parameters: Dict = Field(description="Parameters used for the job") status: str = Field(description="Job status (PENDING, RUNNING, DONE, FAILED, KILLED)") exit_code: Optional[int] = Field(description="Exit code of the job", default=None) @@ -498,6 +500,7 @@ class Job(BaseModel): post_run: Optional[str] = Field(description="Script run after the main command", default=None) pull_latest: bool = Field(description="Whether pull latest was enabled", default=False) cluster_job_id: Optional[str] = Field(description="Cluster-assigned job ID", default=None) + service_url: Optional[str] = Field(description="URL of the running service (for service-type jobs)", default=None) created_at: datetime = Field(description="When the job was created") started_at: Optional[datetime] = Field(description="When the job started running", default=None) finished_at: Optional[datetime] = Field(description="When the job finished", default=None) diff --git a/fileglancer/server.py b/fileglancer/server.py index 0b771f94..b82d95a2 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1728,6 +1728,7 @@ def _convert_job(db_job: db.JobDB, include_files: bool = False) -> Job: manifest_path=db_job.manifest_path, entry_point_id=db_job.entry_point_id, entry_point_name=db_job.entry_point_name, + entry_point_type=getattr(db_job, 'entry_point_type', 'job'), parameters=db_job.parameters, status=db_job.status, exit_code=db_job.exit_code, @@ -1737,6 +1738,7 @@ def _convert_job(db_job: db.JobDB, include_files: bool = False) -> Job: post_run=db_job.post_run, pull_latest=db_job.pull_latest, cluster_job_id=db_job.cluster_job_id, + service_url=apps_module.get_service_url(db_job), created_at=db_job.created_at, started_at=db_job.started_at, finished_at=db_job.finished_at, diff --git a/frontend/src/components/AppLaunch.tsx b/frontend/src/components/AppLaunch.tsx index 7aac1214..a3cae782 100644 --- a/frontend/src/components/AppLaunch.tsx +++ b/frontend/src/components/AppLaunch.tsx @@ -119,7 +119,8 @@ export default function AppLaunch() { pre_run: preRun, post_run: postRun }); - toast.success('Job submitted'); + const isService = selectedEntryPoint.type === 'service'; + toast.success(isService ? 'Service started' : 'Job submitted'); navigate('/apps/jobs'); } catch (error) { const message = @@ -244,12 +245,19 @@ export default function AppLaunch() { key={ep.id} >
- - {ep.name} - +
+ + {ep.name} + + {ep.type === 'service' ? ( + + Service + + ) : null} +
{ep.description ? ( {ep.description} diff --git a/frontend/src/components/JobDetail.tsx b/frontend/src/components/JobDetail.tsx index 06cae847..e08b6014 100644 --- a/frontend/src/components/JobDetail.tsx +++ b/frontend/src/components/JobDetail.tsx @@ -5,7 +5,9 @@ import { Button, Tabs, Typography } from '@material-tailwind/react'; import { HiOutlineArrowLeft, HiOutlineDownload, - HiOutlineRefresh + HiOutlineExternalLink, + HiOutlineRefresh, + HiOutlineStop } from 'react-icons/hi'; import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; import { @@ -22,7 +24,11 @@ import { } from '@/utils/pathHandling'; import { usePreferencesContext } from '@/contexts/PreferencesContext'; import { useZoneAndFspMapContext } from '@/contexts/ZonesAndFspMapContext'; -import { useJobQuery, useJobFileQuery } from '@/queries/jobsQueries'; +import { + useJobQuery, + useJobFileQuery, + useCancelJobMutation +} from '@/queries/jobsQueries'; function FilePreview({ content, @@ -140,6 +146,11 @@ export default function JobDetail() { const scriptQuery = useJobFileQuery(id, 'script'); const stdoutQuery = useJobFileQuery(id, 'stdout'); const stderrQuery = useJobFileQuery(id, 'stderr'); + const cancelMutation = useCancelJobMutation(); + + const isService = jobQuery.data?.entry_point_type === 'service'; + const isActive = + jobQuery.data?.status === 'PENDING' || jobQuery.data?.status === 'RUNNING'; useEffect(() => { const checkDarkMode = () => { @@ -260,6 +271,64 @@ export default function JobDetail() {
+ {/* Service URL banner */} + {isService && job.status === 'RUNNING' ? ( + job.service_url ? ( +
+ + + + + + Service is running at{' '} + + {job.service_url} + + + + + Open Service + +
+ ) : ( +
+ + + + + + Service is starting up... + +
+ ) + ) : null} + + {/* Stop Service button for active services */} + {isService && isActive ? ( +
+ +
+ ) : null} + {/* Tabs */} diff --git a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx index 56ffe025..76fd95e3 100644 --- a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx +++ b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx @@ -845,7 +845,9 @@ export default function AppLaunchForm({ ? 'Validating...' : submitting ? 'Submitting...' - : 'Submit Job'} + : entryPoint.type === 'service' + ? 'Start Service' + : 'Submit Job'} ); diff --git a/frontend/src/components/ui/Table/appsJobsColumns.tsx b/frontend/src/components/ui/Table/appsJobsColumns.tsx index 9c211ee1..2ccce696 100644 --- a/frontend/src/components/ui/Table/appsJobsColumns.tsx +++ b/frontend/src/components/ui/Table/appsJobsColumns.tsx @@ -97,11 +97,25 @@ export function createAppsJobsColumns( { accessorKey: 'status', header: 'Status', - cell: ({ getValue }) => { + cell: ({ getValue, row }) => { const status = getValue() as Job['status']; + const job = row.original; + const hasServiceUrl = + job.entry_point_type === 'service' && job.service_url; return ( -
+ ); }, @@ -148,6 +162,7 @@ export function createAppsJobsColumns( const job = row.original; const canCancel = job.status === 'PENDING' || job.status === 'RUNNING'; + const isService = job.entry_point_type === 'service'; const menuItems: MenuItem[] = [ { name: 'View Details', @@ -158,7 +173,7 @@ export function createAppsJobsColumns( action: props => props.onRelaunch(props.job) }, { - name: 'Cancel', + name: isService ? 'Stop Service' : 'Cancel', action: props => props.onCancel(props.job.id), shouldShow: canCancel }, diff --git a/frontend/src/shared.types.ts b/frontend/src/shared.types.ts index 68b4cd5d..f0cb6f1c 100644 --- a/frontend/src/shared.types.ts +++ b/frontend/src/shared.types.ts @@ -108,6 +108,7 @@ type AppResourceDefaults = { type AppEntryPoint = { id: string; name: string; + type?: 'job' | 'service'; description?: string; command: string; parameters: AppParameterItem[]; @@ -147,6 +148,7 @@ type Job = { manifest_path: string; entry_point_id: string; entry_point_name: string; + entry_point_type?: 'job' | 'service'; parameters: Record; status: 'PENDING' | 'RUNNING' | 'DONE' | 'FAILED' | 'KILLED'; exit_code?: number; @@ -156,6 +158,7 @@ type Job = { post_run?: string; pull_latest: boolean; cluster_job_id?: string; + service_url?: string; created_at: string; started_at?: string; finished_at?: string; diff --git a/pixi.lock b/pixi.lock index bb5fb65e..bdec2561 100644 --- a/pixi.lock +++ b/pixi.lock @@ -3839,7 +3839,7 @@ packages: - pypi: ./ name: fileglancer version: 2.5.0 - sha256: 2449456fb11db97fa768b4228f62867e530fc3d28f0ec1810a59007173d1c8a3 + sha256: 51e22bdacfdcf2069b48aebcd73d234991bcdecf87da989cd5cd343d506a8bd5 requires_dist: - aiosqlite>=0.21.0 - alembic>=1.17.0 diff --git a/pyproject.toml b/pyproject.toml index bfc32ae3..ec1c9683 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -147,8 +147,8 @@ dev-launch = "pixi run uvicorn fileglancer.server:app --no-access-log --port 787 dev-launch-remote = "pixi run uvicorn fileglancer.server:app --host 0.0.0.0 --port 7878 --reload --ssl-keyfile /opt/certs/cert.key --ssl-certfile /opt/certs/cert.crt" prod-launch-remote = "pixi run uvicorn fileglancer.server:app --workers 10 --host 0.0.0.0 --port 7878 --ssl-keyfile /opt/certs/cert.key --ssl-certfile /opt/certs/cert.crt" dev-launch-secure = "python fileglancer/dev_launch.py" -migrate = "alembic upgrade head" -migrate-create = "alembic revision --autogenerate" +migrate = "alembic -c fileglancer/alembic.ini upgrade head" +migrate-create = "alembic -c fileglancer/alembic.ini revision --autogenerate" stamp-db = "python -m fileglancer.stamp_db" container-rebuild = "npx @devcontainers/cli up --workspace-folder . --remove-existing-container" container-shell = "npx @devcontainers/cli exec --workspace-folder . bash" From f0f6a4ed725ac63139a5ffc2d8b236c350640b69 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 18:08:12 -0500 Subject: [PATCH 03/20] service app development --- fileglancer/apps.py | 24 +++++++++++++++--------- frontend/src/shared.types.ts | 2 ++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/fileglancer/apps.py b/fileglancer/apps.py index 63c2818c..bbca2839 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -648,16 +648,13 @@ async def submit_job( tool_repo_dir = await _ensure_repo_cache(manifest.repo_url, pull=pull_latest) repo_link = work_dir / "repo" repo_link.symlink_to(tool_repo_dir) - cd_target = repo_link + cd_suffix = "repo" else: # Tool code is in the discovery repo — cd into manifest's subdirectory repo_dir = await _ensure_repo_cache(app_url, pull=pull_latest) repo_link = work_dir / "repo" repo_link.symlink_to(repo_dir) - if manifest_path: - cd_target = repo_link / manifest_path - else: - cd_target = repo_link + cd_suffix = f"repo/{manifest_path}" if manifest_path else "repo" # Build environment variable export lines env_lines = "" @@ -669,10 +666,19 @@ async def submit_job( parts.append(f"export {var_name}={shlex.quote(var_value)}") env_lines = "\n".join(parts) + "\n" - # Wrap command with cd into the repo symlink - # Unset PIXI_PROJECT_MANIFEST so pixi uses the repo's own manifest - # instead of inheriting fileglancer's from the dev server environment - script_parts = [f"unset PIXI_PROJECT_MANIFEST\ncd {cd_target}"] + # Set up the script preamble: + # - FG_WORK_DIR: the job's working directory (used by subsequent variables) + # - Unset PIXI_PROJECT_MANIFEST so pixi uses the repo's own manifest + # - SERVICE_URL_PATH: for service-type jobs, where to write the service URL + # - cd into the repo so commands can find project files (pixi.toml, scripts, etc.) + preamble_lines = [ + "unset PIXI_PROJECT_MANIFEST", + f"export FG_WORK_DIR={shlex.quote(str(work_dir))}", + ] + if entry_point.type == "service": + preamble_lines.append('export SERVICE_URL_PATH="$FG_WORK_DIR/service_url"') + preamble_lines.append(f'cd "$FG_WORK_DIR/{cd_suffix}"') + script_parts = ["\n".join(preamble_lines)] if env_lines: script_parts.append(env_lines.rstrip()) if effective_pre_run: diff --git a/frontend/src/shared.types.ts b/frontend/src/shared.types.ts index f0cb6f1c..6b6c5c0e 100644 --- a/frontend/src/shared.types.ts +++ b/frontend/src/shared.types.ts @@ -139,6 +139,8 @@ type UserApp = { type JobFileInfo = { path: string; exists: boolean; + fsp_name?: string; + subpath?: string; }; type Job = { From 6e9651c0b2f991a1cf966d5343f4930360b8b153 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 18:18:26 -0500 Subject: [PATCH 04/20] remove open link --- .../src/components/ui/Table/appsJobsColumns.tsx | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/frontend/src/components/ui/Table/appsJobsColumns.tsx b/frontend/src/components/ui/Table/appsJobsColumns.tsx index 2ccce696..c5c30042 100644 --- a/frontend/src/components/ui/Table/appsJobsColumns.tsx +++ b/frontend/src/components/ui/Table/appsJobsColumns.tsx @@ -97,25 +97,11 @@ export function createAppsJobsColumns( { accessorKey: 'status', header: 'Status', - cell: ({ getValue, row }) => { + cell: ({ getValue }) => { const status = getValue() as Job['status']; - const job = row.original; - const hasServiceUrl = - job.entry_point_type === 'service' && job.service_url; return ( ); }, From cdeba46d9d45e9471a3e0ff86636b67ba0f1243b Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 18:25:26 -0500 Subject: [PATCH 05/20] improve UI --- frontend/src/components/AppLaunch.tsx | 10 ----- frontend/src/components/JobDetail.tsx | 59 +++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/frontend/src/components/AppLaunch.tsx b/frontend/src/components/AppLaunch.tsx index a3cae782..735e219d 100644 --- a/frontend/src/components/AppLaunch.tsx +++ b/frontend/src/components/AppLaunch.tsx @@ -202,16 +202,6 @@ export default function AppLaunch() {
) : manifest && selectedEntryPoint ? ( <> - {manifest.runnables.length > 1 ? ( - - ) : null} { const checkDarkMode = () => { @@ -290,6 +290,17 @@ export default function JobDetail() { {job.service_url} + Service is starting up... + ) ) : null} - {/* Stop Service button for active services */} - {isService && isActive ? ( -
+ {/* Stop Service confirmation dialog */} + setShowStopConfirm(false)} + open={showStopConfirm} + > + + Stop Service + + + Are you sure you want to stop this service? It will be terminated + and the URL will no longer be accessible. + +
+
- ) : null} +
{/* Tabs */} From e7d6fbd7cea49396a02f8cc8c910ca6847dc7d00 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 20:32:07 -0500 Subject: [PATCH 06/20] added docs --- docs/AuthoringApps.md | 106 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index 082d8316..11ea5a4a 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -80,6 +80,7 @@ Each runnable defines a single command that users can launch. If the manifest ha |-------|------|----------|-------------| | `id` | string | yes | Unique identifier (used in CLI flags and URLs, should be URL-safe) | | `name` | string | yes | Display name shown in the UI | +| `type` | string | no | `"job"` (default) for batch jobs or `"service"` for long-running services (see [Services](#services)) | | `description` | string | no | Longer description of what this runnable does | | `command` | string | yes | Base shell command to execute (see [Command Building](#command-building)) | | `parameters` | list of objects | no | Parameter definitions (see [Parameters](#parameters)) | @@ -219,8 +220,9 @@ Users can override these in the UI. If a user provides their own pre/post-run sc The generated job script has the following structure: ```bash +export FG_WORK_DIR='/home/user/.fileglancer/jobs/42-MyApp-convert' unset PIXI_PROJECT_MANIFEST -cd /path/to/repo +cd "$FG_WORK_DIR/repo" # Environment variables export JAVA_HOME='/opt/java' @@ -238,6 +240,8 @@ nextflow run main.nf \ echo "Conversion complete" ``` +`FG_WORK_DIR` is always exported and points to the job's working directory. See [Environment Variables Set by Fileglancer](#environment-variables-set-by-fileglancer) for the full list. + ## Command Building When a job is submitted, Fileglancer constructs the full shell command from the runnable's `command` field and the user-provided parameter values using a two-pass approach: @@ -314,6 +318,106 @@ When a user submits a job: Users can view logs, relaunch with the same parameters, or cancel running jobs from the Fileglancer UI. +## Services + +A **service** is a long-running process (web server, notebook, API, viewer) that runs until the user explicitly stops it. Services are declared with `type: service` on the runnable: + +```yaml +runnables: + - id: notebook + name: JupyterLab + type: service + command: jupyter lab --no-browser --ip=0.0.0.0 --port=0 + resources: + cpus: 4 + memory: "32 GB" + walltime: "08:00" +``` + +### How It Works + +From the cluster's perspective, a service is just a long-running batch job. The difference is in how Fileglancer communicates the service URL to the user: + +1. User launches a service-type runnable → job enters PENDING state +2. Cluster picks it up → RUNNING +3. The service starts, binds a port, and writes its URL to the file at `SERVICE_URL_PATH` +4. On the next poll (every few seconds), Fileglancer reads the file and displays the URL in the UI +5. User clicks "Open Service" → service opens in a new browser tab +6. When done, user clicks "Stop Service" → job is killed and the URL disappears + +### Writing the Service URL + +For service-type runnables, Fileglancer exports `SERVICE_URL_PATH` — the absolute path to a file where your service should write its URL. Your service must write its URL (e.g. `http://hostname:port`) to this file once it is ready to accept connections. + +Example in Python: + +```python +import os, socket + +url = f"http://{socket.gethostname()}:{port}" +service_url_path = os.environ.get("SERVICE_URL_PATH") +if service_url_path: + with open(service_url_path, "w") as f: + f.write(url) +``` + +Example in Bash: + +```bash +echo "http://$(hostname):${PORT}" > "$SERVICE_URL_PATH" +``` + +The URL must start with `http://` or `https://`. Fileglancer validates this before displaying it. If the file doesn't exist or contains an invalid URL, no link is shown. + +### Service Lifecycle + +- **Startup**: The service should write its URL to `SERVICE_URL_PATH` as soon as it is ready. Until the file exists, the UI shows "Service is starting up..." +- **Running**: Fileglancer reads the URL file on each poll. If the URL changes (e.g. port rebind), the UI updates automatically. +- **Shutdown**: When the user clicks "Stop Service", Fileglancer sends a SIGTERM to the job. Services should handle this signal for graceful shutdown. Cleaning up the URL file on exit is good practice but not required — Fileglancer only reads it while the job status is RUNNING. + +### Tips + +- **Port selection**: Use port 0 or auto-detection to avoid conflicts when multiple services run on the same node +- **Walltime**: Set a generous walltime — services run until stopped, but the cluster will kill them if walltime expires. Consider `"08:00"` or longer for interactive sessions +- **Flush output**: If running under a batch scheduler like LSF, Python's stdout may be buffered. Use `flush=True` on print statements or set `PYTHONUNBUFFERED=1` so logs appear in real time + +### Service Example + +```yaml +name: My Viewer +description: Interactive data viewer +version: "1.0" + +runnables: + - id: view + name: Start Viewer + type: service + description: Launch an interactive viewer for browsing datasets + command: pixi run python start_viewer.py + parameters: + - flag: --data-dir + name: Data Directory + type: directory + description: Directory containing datasets to view + required: true + + resources: + cpus: 2 + memory: "8 GB" + walltime: "08:00" +``` + +## Environment Variables Set by Fileglancer + +Fileglancer exports the following environment variables in every job script: + +| Variable | Availability | Description | +|----------|-------------|-------------| +| `FG_WORK_DIR` | All jobs | Absolute path to the job's working directory (contains `repo/` symlink, log files, etc.) | +| `SERVICE_URL_PATH` | Service-type jobs only | Absolute path where the service should write its URL. Equivalent to `$FG_WORK_DIR/service_url` | + +These are available to `pre_run` scripts, the main command, and `post_run` scripts. + ## Full Example ```yaml From df3dc20af1ea96b907c604494b25ddc5dcd50216 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 20:38:33 -0500 Subject: [PATCH 07/20] ensure git is run without prompts --- fileglancer/apps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fileglancer/apps.py b/fileglancer/apps.py index bbca2839..61721aa0 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -67,12 +67,14 @@ async def _run_git(args: list[str], timeout: int = 60): Raises ValueError with a readable message on failure. """ + env = {**os.environ, "GIT_TERMINAL_PROMPT": "0"} try: proc = await asyncio.wait_for( asyncio.create_subprocess_exec( *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, + env=env, ), timeout=timeout, ) From 869fbd92b212149b5f7b857ba6015a0319ab926d Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 20:47:04 -0500 Subject: [PATCH 08/20] fix timezone bug --- .../src/components/ui/Table/appsJobsColumns.tsx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/ui/Table/appsJobsColumns.tsx b/frontend/src/components/ui/Table/appsJobsColumns.tsx index c5c30042..752c440d 100644 --- a/frontend/src/components/ui/Table/appsJobsColumns.tsx +++ b/frontend/src/components/ui/Table/appsJobsColumns.tsx @@ -7,12 +7,19 @@ import { formatDateString } from '@/utils'; import type { MenuItem } from '@/components/ui/Menus/FgMenuItems'; import type { Job } from '@/shared.types'; +function parseAsUtc(dateStr: string): number { + // Server timestamps are UTC but may lack a "Z" suffix + if (!/Z$|[+-]\d{2}:\d{2}$/.test(dateStr)) { + return new Date(dateStr + 'Z').getTime(); + } + return new Date(dateStr).getTime(); +} + function formatDuration(job: Job): string { const start = job.started_at || job.created_at; - const end = job.finished_at || new Date().toISOString(); - const startDate = new Date(start); - const endDate = new Date(end); - const diffMs = endDate.getTime() - startDate.getTime(); + const startMs = parseAsUtc(start); + const endMs = job.finished_at ? parseAsUtc(job.finished_at) : Date.now(); + const diffMs = endMs - startMs; if (diffMs < 0) { return '-'; From 957c64f8a638de36d93ec278d7c83cb9334650a6 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Mon, 23 Feb 2026 21:50:13 -0500 Subject: [PATCH 09/20] fix timestamp bugs --- fileglancer/apps.py | 8 +++-- fileglancer/server.py | 19 +++++++++-- .../components/ui/Table/appsJobsColumns.tsx | 18 ++++------ pixi.lock | 34 +++++++++---------- pyproject.toml | 2 +- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/fileglancer/apps.py b/fileglancer/apps.py index 61721aa0..76a9e72a 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -502,11 +502,15 @@ async def _reconcile_jobs(settings): # Terminal transitions are handled by the on_exit callback. new_status = _map_status(tracked.status) if new_status != db_job.status: + # Only store finished_at for terminal states. LSF may report + # a FINISH_TIME for running jobs (projected walltime end), + # which we must not store as an actual finish time. + is_terminal = new_status in ("DONE", "FAILED", "KILLED") db.update_job_status( session, db_job.id, new_status, - exit_code=tracked.exit_code, + exit_code=tracked.exit_code if is_terminal else None, started_at=tracked.start_time, - finished_at=tracked.finish_time, + finished_at=tracked.finish_time if is_terminal else None, ) logger.info(f"Job {db_job.id} status updated: {db_job.status} -> {new_status}") diff --git a/fileglancer/server.py b/fileglancer/server.py index b82d95a2..7b26d39c 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1716,6 +1716,19 @@ async def get_job_file(job_id: int, except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) + def _ensure_utc(dt: Optional[datetime]) -> Optional[datetime]: + """Re-attach UTC timezone to naive datetimes from the DB. + + SQLAlchemy's DateTime column strips tzinfo, so datetimes come back + naive even though they were stored as UTC. Re-attaching ensures + Pydantic serializes with '+00:00' so JS parses them correctly. + """ + if dt is None: + return None + if dt.tzinfo is None: + return dt.replace(tzinfo=UTC) + return dt + def _convert_job(db_job: db.JobDB, include_files: bool = False) -> Job: """Convert a database JobDB to a Pydantic Job model.""" files = None @@ -1739,9 +1752,9 @@ def _convert_job(db_job: db.JobDB, include_files: bool = False) -> Job: pull_latest=db_job.pull_latest, cluster_job_id=db_job.cluster_job_id, service_url=apps_module.get_service_url(db_job), - created_at=db_job.created_at, - started_at=db_job.started_at, - finished_at=db_job.finished_at, + created_at=_ensure_utc(db_job.created_at), + started_at=_ensure_utc(db_job.started_at), + finished_at=_ensure_utc(db_job.finished_at), files=files, ) diff --git a/frontend/src/components/ui/Table/appsJobsColumns.tsx b/frontend/src/components/ui/Table/appsJobsColumns.tsx index 752c440d..6fa523dc 100644 --- a/frontend/src/components/ui/Table/appsJobsColumns.tsx +++ b/frontend/src/components/ui/Table/appsJobsColumns.tsx @@ -7,18 +7,14 @@ import { formatDateString } from '@/utils'; import type { MenuItem } from '@/components/ui/Menus/FgMenuItems'; import type { Job } from '@/shared.types'; -function parseAsUtc(dateStr: string): number { - // Server timestamps are UTC but may lack a "Z" suffix - if (!/Z$|[+-]\d{2}:\d{2}$/.test(dateStr)) { - return new Date(dateStr + 'Z').getTime(); - } - return new Date(dateStr).getTime(); -} - function formatDuration(job: Job): string { - const start = job.started_at || job.created_at; - const startMs = parseAsUtc(start); - const endMs = job.finished_at ? parseAsUtc(job.finished_at) : Date.now(); + if (!job.started_at) { + return '-'; + } + const startMs = new Date(job.started_at).getTime(); + const endMs = job.finished_at + ? new Date(job.finished_at).getTime() + : new Date().getTime(); const diffMs = endMs - startMs; if (diffMs < 0) { diff --git a/pixi.lock b/pixi.lock index bdec2561..54e6333a 100644 --- a/pixi.lock +++ b/pixi.lock @@ -170,7 +170,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b2/f2/889ad4b2408f72fe1a4f6a19491177b30ea7bf1a0fd5f17050ca08cfc882/propcache-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -337,7 +337,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/86/bd/47816020d337f4a746edc42fe8d53669965138f39ee117414c7d7a340cfe/propcache-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -501,7 +501,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/9b/03b04e7d82a5f54fb16113d839f5ea1ede58a61e90edf515f6577c66fa8f/propcache-0.4.1-cp314-cp314-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -665,7 +665,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/fa/89a8ef0468d5833a23fff277b143d0573897cf75bd56670a6d28126c7d68/propcache-0.4.1-cp314-cp314-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -883,7 +883,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b2/f2/889ad4b2408f72fe1a4f6a19491177b30ea7bf1a0fd5f17050ca08cfc882/propcache-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -1092,7 +1092,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/86/bd/47816020d337f4a746edc42fe8d53669965138f39ee117414c7d7a340cfe/propcache-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -1293,7 +1293,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/9b/03b04e7d82a5f54fb16113d839f5ea1ede58a61e90edf515f6577c66fa8f/propcache-0.4.1-cp314-cp314-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -1494,7 +1494,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/fa/89a8ef0468d5833a23fff277b143d0573897cf75bd56670a6d28126c7d68/propcache-0.4.1-cp314-cp314-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/bc/4b/5079831d1b4f37caa13a994f50a9f9d943eef44ea04aa2a2f53d49abbd32/x2s3-1.1.1-py3-none-any.whl @@ -1748,7 +1748,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl - pypi: https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/b2/f2/889ad4b2408f72fe1a4f6a19491177b30ea7bf1a0fd5f17050ca08cfc882/propcache-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl @@ -1996,7 +1996,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/86/bd/47816020d337f4a746edc42fe8d53669965138f39ee117414c7d7a340cfe/propcache-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl @@ -2244,7 +2244,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl - pypi: https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: https://files.pythonhosted.org/packages/65/9b/03b04e7d82a5f54fb16113d839f5ea1ede58a61e90edf515f6577c66fa8f/propcache-0.4.1-cp314-cp314-macosx_10_13_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl @@ -2492,7 +2492,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/b2/fa/89a8ef0468d5833a23fff277b143d0573897cf75bd56670a6d28126c7d68/propcache-0.4.1-cp314-cp314-macosx_11_0_arm64.whl - - pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/48/a2/87a0b61a3078be07ab04ec574ef6c683c764590ed0d2a50d00cbb23aeae7/pydantic_settings_yaml-0.2.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl @@ -3839,7 +3839,7 @@ packages: - pypi: ./ name: fileglancer version: 2.5.0 - sha256: 51e22bdacfdcf2069b48aebcd73d234991bcdecf87da989cd5cd343d506a8bd5 + sha256: f8b3583bb4ce03c14eb05aa2627ce10ca6587782b201d1f8b8498c7901c46815 requires_dist: - aiosqlite>=0.21.0 - alembic>=1.17.0 @@ -3855,7 +3855,7 @@ packages: - lxml>=5.3.1 - pandas>=2.3.3 - psycopg2-binary>=2.9.10,<3 - - py-cluster-api>=0.2.3 + - py-cluster-api>=0.2.4 - pydantic-settings>=2.11.0 - pydantic>=2.10.6 - python-jose>=3.5.0,<4 @@ -7452,10 +7452,10 @@ packages: - pkg:pypi/pure-eval?source=hash-mapping size: 16668 timestamp: 1733569518868 -- pypi: https://files.pythonhosted.org/packages/e1/b5/226661a1ca70b1b06500f4c4cc004a90d6bc421fb8bb2446d012511f2578/py_cluster_api-0.2.3-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/58/07/0bf2e004d397f3beca035a53d193e5ed08f37372cac92f5017f9afac0439/py_cluster_api-0.2.4-py3-none-any.whl name: py-cluster-api - version: 0.2.3 - sha256: 44c129325e8b440ff3c2735e044dd88af0191839ece3c587a23d0d53bce9021f + version: 0.2.4 + sha256: 6f0554394e2b36429641a40ddb8391636a87033c67f12ad5af515a5ede1ba0df requires_dist: - pyyaml - build ; extra == 'release' diff --git a/pyproject.toml b/pyproject.toml index ec1c9683..16e92507 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "sqlalchemy >=2.0.44", "uvicorn >=0.38.0", "x2s3 >=1.1.1", - "py-cluster-api >=0.2.3" + "py-cluster-api >=0.2.4" ] [project.scripts] From 7c841a86064eefcc022c9f2af9dbe5972b37a662 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 24 Feb 2026 07:56:42 -0500 Subject: [PATCH 10/20] default to local executor --- docs/config.yaml.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config.yaml.template b/docs/config.yaml.template index 5f44863f..9b9fe167 100644 --- a/docs/config.yaml.template +++ b/docs/config.yaml.template @@ -129,8 +129,8 @@ session_cookie_secure: true # Mirrors py-cluster-api ClusterConfig - all fields are optional # See: https://github.com/JaneliaSciComp/py-cluster-api # -# cluster: -# executor: local # "local" or "lsf" +cluster: + executor: local # "local" or "lsf" # job_name_prefix: fg # Prefix for cluster job names. REQUIRED for job # # reconnection after server restarts. Without this, # # active jobs will not be re-tracked and will From 247402ad254124b3dfb4bafb18df109560478a94 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Tue, 24 Feb 2026 11:46:47 -0500 Subject: [PATCH 11/20] added more cluster options, override for extra args --- fileglancer/apps.py | 18 +- fileglancer/model.py | 2 + fileglancer/server.py | 8 + frontend/src/components/AppLaunch.tsx | 33 ++-- frontend/src/components/Preferences.tsx | 2 + .../components/ui/AppsPage/AppLaunchForm.tsx | 155 ++++++++++++++++-- .../ui/PreferencesPage/JobOptions.tsx | 61 +++++++ frontend/src/contexts/PreferencesContext.tsx | 18 ++ frontend/src/queries/jobsQueries.ts | 37 +++++ frontend/src/queries/preferencesQueries.ts | 5 +- frontend/src/shared.types.ts | 2 + 11 files changed, 312 insertions(+), 29 deletions(-) create mode 100644 frontend/src/components/ui/PreferencesPage/JobOptions.tsx diff --git a/fileglancer/apps.py b/fileglancer/apps.py index 76a9e72a..cb9d5557 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -573,6 +573,7 @@ async def submit_job( entry_point_id: str, parameters: dict, resources: Optional[dict] = None, + extra_args: Optional[str] = None, pull_latest: bool = False, manifest_path: str = "", env: Optional[dict] = None, @@ -605,8 +606,11 @@ async def submit_job( # Build command command = build_command(entry_point, parameters) - # Build resource spec - resource_spec = _build_resource_spec(entry_point, resources, settings) + # Build resource spec (extra_args passed separately, not from manifest) + overrides = dict(resources) if resources else {} + if extra_args is not None: + overrides["extra_args"] = extra_args + resource_spec = _build_resource_spec(entry_point, overrides or None, settings) # Merge env/pre_run/post_run: manifest defaults overridden by user values merged_env = dict(entry_point.env or {}) @@ -623,6 +627,7 @@ async def submit_job( "memory": resource_spec.memory, "walltime": resource_spec.walltime, "queue": resource_spec.queue, + "extra_args": " ".join(resource_spec.extra_args) if resource_spec.extra_args else None, } with db.get_db_session(settings.db_url) as session: @@ -739,8 +744,12 @@ def _build_resource_spec(entry_point: AppEntryPoint, overrides: Optional[dict], memory = entry_point.resources.memory if entry_point.resources.walltime is not None: walltime = entry_point.resources.walltime + if entry_point.resources.queue is not None: + queue = entry_point.resources.queue # Apply user overrides + # Note: extra_args replaces (not extends) config defaults via ResourceSpec + extra_args = None if overrides: if overrides.get("cpus") is not None: cpus = overrides["cpus"] @@ -748,12 +757,17 @@ def _build_resource_spec(entry_point: AppEntryPoint, overrides: Optional[dict], memory = overrides["memory"] if overrides.get("walltime") is not None: walltime = overrides["walltime"] + if overrides.get("queue") is not None: + queue = overrides["queue"] + if overrides.get("extra_args") is not None: + extra_args = [overrides["extra_args"]] return ResourceSpec( cpus=cpus, memory=memory, walltime=walltime, queue=queue, + extra_args=extra_args, ) diff --git a/fileglancer/model.py b/fileglancer/model.py index 45fde4b5..4609c04d 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -376,6 +376,7 @@ class AppResourceDefaults(BaseModel): cpus: Optional[int] = Field(description="Number of CPUs", default=None) memory: Optional[str] = Field(description="Memory allocation (e.g. '16 GB')", default=None) walltime: Optional[str] = Field(description="Wall time limit (e.g. '04:00')", default=None) + queue: Optional[str] = Field(description="Cluster queue/partition name", default=None) class AppEntryPoint(BaseModel): @@ -514,6 +515,7 @@ class JobSubmitRequest(BaseModel): entry_point_id: str = Field(description="Entry point to execute") parameters: Dict = Field(description="Parameter values keyed by parameter key") resources: Optional[AppResourceDefaults] = Field(description="Resource overrides", default=None) + extra_args: Optional[str] = Field(description="Extra CLI args for the submit command (replaces config defaults)", default=None) pull_latest: bool = Field( description="Pull latest code from GitHub before running", default=False, diff --git a/fileglancer/server.py b/fileglancer/server.py index 7b26d39c..4711e326 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1634,6 +1634,13 @@ async def validate_paths(body: PathValidationRequest, errors[param_key] = f"Path is not accessible: {normalized}" return PathValidationResponse(errors=errors) + @app.get("/api/cluster-defaults", + description="Get cluster configuration defaults") + async def get_cluster_defaults(): + return { + "extra_args": " ".join(settings.cluster.extra_args), + } + @app.post("/api/jobs", response_model=Job, description="Submit a new job") async def submit_job(body: JobSubmitRequest, @@ -1649,6 +1656,7 @@ async def submit_job(body: JobSubmitRequest, entry_point_id=body.entry_point_id, parameters=body.parameters, resources=resources_dict, + extra_args=body.extra_args, pull_latest=body.pull_latest, manifest_path=body.manifest_path, env=body.env, diff --git a/frontend/src/components/AppLaunch.tsx b/frontend/src/components/AppLaunch.tsx index 735e219d..222e8d37 100644 --- a/frontend/src/components/AppLaunch.tsx +++ b/frontend/src/components/AppLaunch.tsx @@ -58,6 +58,10 @@ export default function AppLaunch() { const relaunchResources = relaunchState?.resources as | AppResourceDefaults | undefined; + // extra_args stored in resources dict from previous job + const relaunchExtraArgs = relaunchState?.resources?.extra_args as + | string + | undefined; const relaunchEnv = relaunchState?.env; const relaunchPreRun = relaunchState?.pre_run; const relaunchPostRun = relaunchState?.post_run; @@ -99,6 +103,7 @@ export default function AppLaunch() { const handleSubmit = async ( parameters: Record, resources?: AppResourceDefaults, + extraArgs?: string, pullLatest?: boolean, env?: Record, preRun?: string, @@ -114,6 +119,7 @@ export default function AppLaunch() { entry_point_id: selectedEntryPoint.id, parameters, resources, + extra_args: extraArgs, pull_latest: pullLatest, env, pre_run: preRun, @@ -201,20 +207,19 @@ export default function AppLaunch() { {manifestMutation.error?.message || 'Unknown error'}
) : manifest && selectedEntryPoint ? ( - <> - - + ) : manifest ? (
diff --git a/frontend/src/components/Preferences.tsx b/frontend/src/components/Preferences.tsx index c4d553f2..7961f59c 100644 --- a/frontend/src/components/Preferences.tsx +++ b/frontend/src/components/Preferences.tsx @@ -5,6 +5,7 @@ import toast from 'react-hot-toast'; import { usePreferencesContext } from '@/contexts/PreferencesContext'; import DataLinkOptions from '@/components/ui/PreferencesPage/DataLinkOptions'; import DisplayOptions from '@/components/ui/PreferencesPage/DisplayOptions'; +import JobOptions from '@/components/ui/PreferencesPage/JobOptions'; import NeuroglancerOptions from '@/components/ui/PreferencesPage/NeuroglancerOptions'; export default function Preferences() { @@ -122,6 +123,7 @@ export default function Preferences() { + diff --git a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx index 76fd95e3..d11a7f44 100644 --- a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx +++ b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx @@ -10,7 +10,9 @@ import { } from 'react-icons/hi'; import FileSelectorButton from '@/components/ui/BrowsePage/FileSelector/FileSelectorButton'; +import { usePreferencesContext } from '@/contexts/PreferencesContext'; import { validatePaths } from '@/queries/appsQueries'; +import { useClusterDefaultsQuery } from '@/queries/jobsQueries'; import { convertBackToForwardSlash } from '@/utils/pathHandling'; import { flattenParameters, isParameterSection } from '@/shared.types'; import type { @@ -27,6 +29,7 @@ interface AppLaunchFormProps { readonly onSubmit: ( parameters: Record, resources?: AppResourceDefaults, + extraArgs?: string, pullLatest?: boolean, env?: Record, preRun?: string, @@ -35,6 +38,7 @@ interface AppLaunchFormProps { readonly submitting: boolean; readonly initialValues?: Record; readonly initialResources?: AppResourceDefaults; + readonly initialExtraArgs?: string; readonly initialEnv?: Record; readonly initialPreRun?: string; readonly initialPostRun?: string; @@ -411,6 +415,61 @@ function ResourcesSectionContent({ ); } +function SubmitOptionsSectionContent({ + resources, + setResources, + extraArgs, + setExtraArgs +}: { + readonly resources: AppResourceDefaults; + readonly setResources: Dispatch>; + readonly extraArgs: string; + readonly setExtraArgs: Dispatch>; +}) { + const inputClass = + 'w-full p-2 text-foreground border rounded-sm focus:outline-none bg-background border-primary-light focus:border-primary'; + + return ( +
+
+ + + Cluster queue/partition to submit the job to + + + setResources(prev => ({ + ...prev, + queue: e.target.value || undefined + })) + } + placeholder="e.g. normal" + type="text" + value={resources.queue ?? ''} + /> +
+
+ + + Additional CLI arguments for the submit command + + setExtraArgs(e.target.value)} + placeholder='e.g. -P your_project -R "select[mem>8000]"' + type="text" + value={extraArgs} + /> +
+
+ ); +} + function EnvironmentTabContent({ envVars, setEnvVars, @@ -418,8 +477,6 @@ function EnvironmentTabContent({ setPreRun, postRun, setPostRun, - resources, - setResources, openEnvSections, setOpenEnvSections }: { @@ -429,8 +486,6 @@ function EnvironmentTabContent({ readonly setPreRun: Dispatch>; readonly postRun: string; readonly setPostRun: Dispatch>; - readonly resources: AppResourceDefaults; - readonly setResources: Dispatch>; readonly openEnvSections: string[]; readonly setOpenEnvSections: Dispatch>; }) { @@ -444,7 +499,7 @@ function EnvironmentTabContent({ > -
Environment
+
Environment
+ + ); +} +function ClusterTabContent({ + resources, + setResources, + extraArgs, + setExtraArgs, + openClusterSections, + setOpenClusterSections +}: { + readonly resources: AppResourceDefaults; + readonly setResources: Dispatch>; + readonly extraArgs: string; + readonly setExtraArgs: Dispatch>; + readonly openClusterSections: string[]; + readonly setOpenClusterSections: Dispatch>; +}) { + return ( + > + } + type="multiple" + value={openClusterSections} + > -
Resources
+
Resources
@@ -479,6 +560,27 @@ function EnvironmentTabContent({ />
+ + + +
+ Submit Options +
+ +
+ + + +
); } @@ -490,11 +592,14 @@ export default function AppLaunchForm({ submitting, initialValues: externalValues, initialResources, + initialExtraArgs: externalExtraArgs, initialEnv, initialPreRun, initialPostRun, initialPullLatest }: AppLaunchFormProps) { + const { defaultExtraArgs } = usePreferencesContext(); + const clusterDefaultsQuery = useClusterDefaultsQuery(); const allParams = flattenParameters(entryPoint.parameters); // Initialize parameter values: external values override defaults @@ -513,6 +618,11 @@ export default function AppLaunchForm({ .filter(item => isParameterSection(item) && !item.collapsed) .map(item => (item as AppParameterSection).section); + // extra_args priority: relaunch > user preference > config.yaml + const configExtraArgs = clusterDefaultsQuery.data?.extra_args ?? ''; + const resolvedExtraArgs = + externalExtraArgs ?? (defaultExtraArgs || configExtraArgs); + const [values, setValues] = useState>(startingValues); const [errors, setErrors] = useState>({}); const [activeTab, setActiveTab] = useState('parameters'); @@ -523,9 +633,11 @@ export default function AppLaunchForm({ initialResources ?? { cpus: entryPoint.resources?.cpus, memory: entryPoint.resources?.memory, - walltime: entryPoint.resources?.walltime + walltime: entryPoint.resources?.walltime, + queue: entryPoint.resources?.queue } ); + const [extraArgs, setExtraArgs] = useState(resolvedExtraArgs); // Environment tab state — relaunch values override entry point defaults const [envVars, setEnvVars] = useState(() => { @@ -542,8 +654,11 @@ export default function AppLaunchForm({ initialPostRun ?? entryPoint.post_run ?? '' ); const [openEnvSections, setOpenEnvSections] = useState([ - 'environment', - 'resources' + 'environment' + ]); + const [openClusterSections, setOpenClusterSections] = useState([ + 'resources', + 'submitOptions' ]); const handleChange = (paramId: string, value: unknown) => { @@ -671,7 +786,10 @@ export default function AppLaunchForm({ // Only pass resources if user provided values const hasResourceOverrides = - resources.cpus || resources.memory || resources.walltime; + resources.cpus || + resources.memory || + resources.walltime || + resources.queue; // Convert envVars array to Record, filtering empty keys const envRecord: Record = {}; @@ -685,6 +803,7 @@ export default function AppLaunchForm({ await onSubmit( params, hasResourceOverrides ? resources : undefined, + extraArgs.trim() || undefined, pullLatest, hasEnv ? envRecord : undefined, preRun.trim() || undefined, @@ -718,6 +837,9 @@ export default function AppLaunchForm({ Environment + + Cluster + @@ -817,11 +939,20 @@ export default function AppLaunchForm({ openEnvSections={openEnvSections} postRun={postRun} preRun={preRun} - resources={resources} setEnvVars={setEnvVars} setOpenEnvSections={setOpenEnvSections} setPostRun={setPostRun} setPreRun={setPreRun} + /> + + + + diff --git a/frontend/src/components/ui/PreferencesPage/JobOptions.tsx b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx new file mode 100644 index 00000000..f6317eff --- /dev/null +++ b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx @@ -0,0 +1,61 @@ +import { useState } from 'react'; + +import { Typography } from '@material-tailwind/react'; +import toast from 'react-hot-toast'; + +import { usePreferencesContext } from '@/contexts/PreferencesContext'; + +export default function JobOptions() { + const { defaultExtraArgs, updateDefaultExtraArgs } = usePreferencesContext(); + + const [localValue, setLocalValue] = useState(defaultExtraArgs); + const [saving, setSaving] = useState(false); + + // Track whether the local value differs from the saved preference + const isDirty = localValue !== defaultExtraArgs; + + const handleSave = async () => { + setSaving(true); + const result = await updateDefaultExtraArgs(localValue.trim()); + setSaving(false); + if (result.success) { + toast.success('Default extra arguments saved'); + } else { + toast.error(result.error); + } + }; + + return ( +
+ Jobs +
+ + + Additional CLI arguments appended to every job submit command. Can be + overridden per job on the Cluster tab. + + setLocalValue(e.target.value)} + placeholder="e.g. -P your_project" + type="text" + value={localValue} + /> + +
+
+ ); +} diff --git a/frontend/src/contexts/PreferencesContext.tsx b/frontend/src/contexts/PreferencesContext.tsx index 877e484f..3c75c28d 100644 --- a/frontend/src/contexts/PreferencesContext.tsx +++ b/frontend/src/contexts/PreferencesContext.tsx @@ -53,6 +53,7 @@ type PreferencesContextType = { useLegacyMultichannelApproach: boolean; isFilteredByGroups: boolean; showTutorial: boolean; + defaultExtraArgs: string; // Favorites zoneFavorites: Zone[]; @@ -80,6 +81,7 @@ type PreferencesContextType = { toggleUseLegacyMultichannelApproach: () => Promise>; toggleFilterByGroups: () => Promise>; toggleShowTutorial: () => Promise>; + updateDefaultExtraArgs: (args: string) => Promise>; handleFavoriteChange: ( item: Zone | FileSharePath | FolderFavorite, type: 'zone' | 'fileSharePath' | 'folder' @@ -242,6 +244,20 @@ export const PreferencesProvider = ({ ); }; + const updateDefaultExtraArgs = async ( + args: string + ): Promise> => { + try { + await updatePreferenceMutation.mutateAsync({ + key: 'defaultExtraArgs', + value: args + }); + return createSuccess(undefined); + } catch (error) { + return handleError(error); + } + }; + function updatePreferenceList( key: string, itemToUpdate: T, @@ -500,6 +516,7 @@ export const PreferencesProvider = ({ preferencesQuery.data?.useLegacyMultichannelApproach || false, isFilteredByGroups: preferencesQuery.data?.isFilteredByGroups ?? true, showTutorial: preferencesQuery.data?.showTutorial ?? true, + defaultExtraArgs: preferencesQuery.data?.defaultExtraArgs || '', // Favorites zoneFavorites: preferencesQuery.data?.zoneFavorites || [], @@ -526,6 +543,7 @@ export const PreferencesProvider = ({ toggleUseLegacyMultichannelApproach, toggleFilterByGroups, toggleShowTutorial, + updateDefaultExtraArgs, handleFavoriteChange, handleContextMenuFavorite }; diff --git a/frontend/src/queries/jobsQueries.ts b/frontend/src/queries/jobsQueries.ts index c53cc319..87b8ed10 100644 --- a/frontend/src/queries/jobsQueries.ts +++ b/frontend/src/queries/jobsQueries.ts @@ -8,8 +8,18 @@ import { } from '@/queries/queryUtils'; import type { Job, JobSubmitRequest } from '@/shared.types'; +// --- Types --- + +type ClusterDefaults = { + extra_args: string; +}; + // --- Query Keys --- +export const clusterDefaultsQueryKeys = { + all: ['cluster-defaults'] as const +}; + export const jobsQueryKeys = { all: ['cluster-jobs'] as const, list: () => ['cluster-jobs', 'list'] as const, @@ -18,6 +28,22 @@ export const jobsQueryKeys = { // --- Fetch Helpers --- +async function fetchClusterDefaults( + signal?: AbortSignal +): Promise { + const response = await sendFetchRequest( + '/api/cluster-defaults', + 'GET', + undefined, + { signal } + ); + const data = await getResponseJsonOrError(response); + if (!response.ok) { + throwResponseNotOkError(response, data); + } + return data as ClusterDefaults; +} + async function fetchJobs(signal?: AbortSignal): Promise { const response = await sendFetchRequest('/api/jobs', 'GET', undefined, { signal @@ -67,6 +93,17 @@ async function fetchJobFile( // --- Query Hooks --- +export function useClusterDefaultsQuery(): UseQueryResult< + ClusterDefaults, + Error +> { + return useQuery({ + queryKey: clusterDefaultsQueryKeys.all, + queryFn: ({ signal }) => fetchClusterDefaults(signal), + staleTime: 1000 * 60 * 60 // 1 hour — cluster config rarely changes + }); +} + export function useJobsQuery(): UseQueryResult { return useQuery({ queryKey: jobsQueryKeys.list(), diff --git a/frontend/src/queries/preferencesQueries.ts b/frontend/src/queries/preferencesQueries.ts index a5962069..a1aa9bec 100644 --- a/frontend/src/queries/preferencesQueries.ts +++ b/frontend/src/queries/preferencesQueries.ts @@ -38,6 +38,7 @@ type PreferencesApiResponse = { useLegacyMultichannelApproach?: { value: boolean }; isFilteredByGroups?: { value: boolean }; showTutorial?: { value: boolean }; + defaultExtraArgs?: { value: string }; zone?: { value: ZonePreference[] }; fileSharePath?: { value: FileSharePathPreference[] }; folder?: { value: FolderPreference[] }; @@ -68,6 +69,7 @@ export type PreferencesQueryData = { useLegacyMultichannelApproach: boolean; isFilteredByGroups: boolean; showTutorial: boolean; + defaultExtraArgs: string; }; /** @@ -232,7 +234,8 @@ const createTransformPreferences = ( useLegacyMultichannelApproach: rawData.useLegacyMultichannelApproach?.value || false, isFilteredByGroups: rawData.isFilteredByGroups?.value ?? true, - showTutorial: rawData.showTutorial?.value ?? true + showTutorial: rawData.showTutorial?.value ?? true, + defaultExtraArgs: rawData.defaultExtraArgs?.value || '' }; }; }; diff --git a/frontend/src/shared.types.ts b/frontend/src/shared.types.ts index 6b6c5c0e..635b0004 100644 --- a/frontend/src/shared.types.ts +++ b/frontend/src/shared.types.ts @@ -103,6 +103,7 @@ type AppResourceDefaults = { cpus?: number; memory?: string; walltime?: string; + queue?: string; }; type AppEntryPoint = { @@ -173,6 +174,7 @@ type JobSubmitRequest = { entry_point_id: string; parameters: Record; resources?: AppResourceDefaults; + extra_args?: string; pull_latest?: boolean; env?: Record; pre_run?: string; From d5215e7861ae441b7a380d5ca0440a8dc7981b52 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Wed, 25 Feb 2026 11:06:51 -0500 Subject: [PATCH 12/20] fixed _MANIFEST_FILENAME bug --- fileglancer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fileglancer/server.py b/fileglancer/server.py index 4711e326..c6418aa9 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1520,7 +1520,7 @@ async def add_user_app(body: AppAddRequest, raise HTTPException(status_code=400, detail=f"Failed to clone or scan repo: {str(e)}") if not discovered: - filenames = ", ".join(apps_module._MANIFEST_FILENAMES) + filenames = apps_module._MANIFEST_FILENAME raise HTTPException( status_code=404, detail=f"No manifest files found ({filenames}). " From e29288eee824816c0c1bb75f17fdd1670e04eee1 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Thu, 26 Feb 2026 16:10:58 -0500 Subject: [PATCH 13/20] added conda support for apps --- docs/AuthoringApps.md | 49 ++++++++- fileglancer/apps.py | 13 +++ fileglancer/model.py | 28 +++++- frontend/src/shared.types.ts | 1 + tests/test_apps.py | 188 +++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 tests/test_apps.py diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index 11ea5a4a..eb1f14ad 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -64,9 +64,10 @@ requirements: - "pixi>=0.40" - npm - "maven>=3.9" + - miniforge ``` -**Supported tools:** `pixi`, `npm`, `maven` +**Supported tools:** `pixi`, `npm`, `maven`, `miniforge` **Supported version operators:** `>=`, `<=`, `!=`, `==`, `>`, `<` @@ -88,6 +89,7 @@ Each runnable defines a single command that users can launch. If the manifest ha | `env` | object | no | Default environment variables to export (see [Environment Variables](#environment-variables)) | | `pre_run` | string | no | Shell script to run before the main command (see [Pre/Post-Run Scripts](#prepost-run-scripts)) | | `post_run` | string | no | Shell script to run after the main command (see [Pre/Post-Run Scripts](#prepost-run-scripts)) | +| `conda_env` | string | no | Conda environment name or absolute path to activate before running (see [Conda Environments](#conda-environments)) | ### Parameters @@ -220,10 +222,14 @@ Users can override these in the UI. If a user provides their own pre/post-run sc The generated job script has the following structure: ```bash -export FG_WORK_DIR='/home/user/.fileglancer/jobs/42-MyApp-convert' unset PIXI_PROJECT_MANIFEST +export FG_WORK_DIR='/home/user/.fileglancer/jobs/42-MyApp-convert' cd "$FG_WORK_DIR/repo" +# Conda activation (if conda_env is set) +eval "$(conda shell.bash hook)" +conda activate myenv + # Environment variables export JAVA_HOME='/opt/java' export NXF_SINGULARITY_CACHEDIR='/scratch/singularity' @@ -242,6 +248,45 @@ echo "Conversion complete" `FG_WORK_DIR` is always exported and points to the job's working directory. See [Environment Variables Set by Fileglancer](#environment-variables-set-by-fileglancer) for the full list. +### Conda Environments + +The `conda_env` field specifies a conda environment to activate before running the command. This requires `miniforge` (or any conda distribution providing the `conda` binary) in the `requirements` list. + +The value can be either: +- **An environment name** (e.g. `myenv`): must match `[a-zA-Z0-9_.-]+` +- **An absolute path** (e.g. `/opt/envs/myenv`): must not contain shell metacharacters + +```yaml +name: My Analysis Tool +requirements: + - miniforge +runnables: + - id: analyze + name: Run Analysis + command: python analyze.py + conda_env: my-analysis-env + parameters: + - flag: --input + name: Input + type: file + required: true +``` + +When `conda_env` is set, the generated script initializes conda and activates the environment before any env vars, pre_run, or the main command: + +```bash +eval "$(conda shell.bash hook)" +conda activate my-analysis-env +``` + +> **Tip:** If the conda environment needs to be created before use (e.g. from an `environment.yml`), use `pre_run` to create it first: +> +> ```yaml +> conda_env: my-tool-env +> pre_run: | +> conda env create -f environment.yml -n my-tool-env --yes 2>/dev/null || true +> ``` + ## Command Building When a job is submitted, Fileglancer constructs the full shell command from the runnable's `command` field and the user-provided parameter values using a two-pass approach: diff --git a/fileglancer/apps.py b/fileglancer/apps.py index cb9d5557..b6a4c916 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -188,6 +188,10 @@ async def fetch_app_manifest(url: str, manifest_path: str = "") -> AppManifest: "version_args": ["mvn", "--version"], "version_pattern": r"Apache Maven (\S+)", }, + "miniforge": { + "version_args": ["conda", "--version"], + "version_pattern": r"conda (\S+)", + }, } _REQ_PATTERN = re.compile(r"^([a-zA-Z][a-zA-Z0-9_-]*)\s*((?:>=|<=|!=|==|>|<)\s*\S+)?$") @@ -690,6 +694,15 @@ async def submit_job( preamble_lines.append('export SERVICE_URL_PATH="$FG_WORK_DIR/service_url"') preamble_lines.append(f'cd "$FG_WORK_DIR/{cd_suffix}"') script_parts = ["\n".join(preamble_lines)] + + # Conda environment activation + if entry_point.conda_env: + conda_activation = ( + 'eval "$(conda shell.bash hook)"\n' + f'conda activate {shlex.quote(entry_point.conda_env)}' + ) + script_parts.append(conda_activation) + if env_lines: script_parts.append(env_lines.rstrip()) if effective_pre_run: diff --git a/fileglancer/model.py b/fileglancer/model.py index 4609c04d..da74d2e3 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -391,6 +391,29 @@ class AppEntryPoint(BaseModel): env: Optional[Dict[str, str]] = Field(description="Default environment variables", default=None) pre_run: Optional[str] = Field(description="Script to run before the main command", default=None) post_run: Optional[str] = Field(description="Script to run after the main command", default=None) + conda_env: Optional[str] = Field( + description="Conda environment name or path to activate before running", + default=None, + ) + + @field_validator("conda_env") + @classmethod + def validate_conda_env(cls, v): + if v is None: + return v + if v.startswith("/"): + # Absolute path: reject shell metacharacters + if _CONDA_ENV_PATH_FORBIDDEN.search(v): + raise ValueError( + f"conda_env path contains forbidden characters: {v!r}" + ) + else: + # Name: must be alphanumeric, dots, dashes, underscores + if not _CONDA_ENV_NAME_PATTERN.match(v): + raise ValueError( + f"conda_env name must match [a-zA-Z0-9_.-]+, got: {v!r}" + ) + return v def flat_parameters(self) -> List[AppParameter]: """Return a flat list of all parameters, traversing sections.""" @@ -421,7 +444,10 @@ def generate_parameter_keys(self): return self -SUPPORTED_TOOLS = {"pixi", "npm", "maven"} +SUPPORTED_TOOLS = {"pixi", "npm", "maven", "miniforge"} + +_CONDA_ENV_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9_.-]+$') +_CONDA_ENV_PATH_FORBIDDEN = re.compile(r'[;&|`$(){}!<>\n\r]') class AppManifest(BaseModel): diff --git a/frontend/src/shared.types.ts b/frontend/src/shared.types.ts index 635b0004..28661734 100644 --- a/frontend/src/shared.types.ts +++ b/frontend/src/shared.types.ts @@ -117,6 +117,7 @@ type AppEntryPoint = { env?: Record; pre_run?: string; post_run?: string; + conda_env?: string; }; type AppManifest = { diff --git a/tests/test_apps.py b/tests/test_apps.py new file mode 100644 index 00000000..23776248 --- /dev/null +++ b/tests/test_apps.py @@ -0,0 +1,188 @@ +"""Tests for apps module: miniforge requirement and conda_env support.""" + +import subprocess +from unittest.mock import patch, MagicMock + +import pytest +from pydantic import ValidationError + +from fileglancer.model import SUPPORTED_TOOLS, AppEntryPoint +from fileglancer.apps import _TOOL_REGISTRY, verify_requirements + + +# --- Model tests --- + +class TestSupportedTools: + def test_miniforge_in_supported_tools(self): + assert "miniforge" in SUPPORTED_TOOLS + + def test_miniforge_in_tool_registry(self): + assert "miniforge" in _TOOL_REGISTRY + entry = _TOOL_REGISTRY["miniforge"] + assert entry["version_args"] == ["conda", "--version"] + assert entry["version_pattern"] == r"conda (\S+)" + + +class TestCondaEnvValidation: + def test_valid_name(self): + ep = AppEntryPoint(id="t", name="T", command="echo", conda_env="myenv") + assert ep.conda_env == "myenv" + + def test_valid_name_with_dots_dashes(self): + ep = AppEntryPoint(id="t", name="T", command="echo", conda_env="my.env-2_test") + assert ep.conda_env == "my.env-2_test" + + def test_valid_absolute_path(self): + ep = AppEntryPoint(id="t", name="T", command="echo", conda_env="/opt/envs/myenv") + assert ep.conda_env == "/opt/envs/myenv" + + def test_none_is_allowed(self): + ep = AppEntryPoint(id="t", name="T", command="echo", conda_env=None) + assert ep.conda_env is None + + def test_default_is_none(self): + ep = AppEntryPoint(id="t", name="T", command="echo") + assert ep.conda_env is None + + def test_rejects_name_with_spaces(self): + with pytest.raises(ValidationError, match="conda_env name must match"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="my env") + + def test_rejects_name_with_semicolon(self): + with pytest.raises(ValidationError, match="conda_env name must match"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="env;rm") + + def test_rejects_path_with_semicolon(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="/opt/envs;rm -rf /") + + def test_rejects_path_with_backtick(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="/opt/`whoami`/env") + + def test_rejects_path_with_dollar(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="/opt/$HOME/env") + + def test_rejects_path_with_pipe(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint(id="t", name="T", command="echo", conda_env="/opt/env|bad") + + +# --- verify_requirements tests --- + +class TestVerifyRequirementsMiniforge: + @patch("fileglancer.apps.shutil.which") + def test_miniforge_found_via_conda(self, mock_which): + """miniforge binary doesn't exist, but conda does — should pass.""" + def which_side_effect(name): + if name == "miniforge": + return None + if name == "conda": + return "/usr/bin/conda" + return None + mock_which.side_effect = which_side_effect + + # No version constraint, so just checking binary existence + verify_requirements(["miniforge"]) + + @patch("fileglancer.apps.shutil.which") + def test_miniforge_not_found(self, mock_which): + mock_which.return_value = None + with pytest.raises(ValueError, match="not installed or not on PATH"): + verify_requirements(["miniforge"]) + + @patch("fileglancer.apps.subprocess.run") + @patch("fileglancer.apps.shutil.which") + def test_miniforge_version_check(self, mock_which, mock_run): + def which_side_effect(name): + if name == "miniforge": + return None + if name == "conda": + return "/usr/bin/conda" + return None + mock_which.side_effect = which_side_effect + + mock_run.return_value = MagicMock( + stdout="conda 24.7.1", stderr="", returncode=0 + ) + verify_requirements(["miniforge>=24.0"]) + + @patch("fileglancer.apps.subprocess.run") + @patch("fileglancer.apps.shutil.which") + def test_miniforge_version_too_old(self, mock_which, mock_run): + def which_side_effect(name): + if name == "miniforge": + return None + if name == "conda": + return "/usr/bin/conda" + return None + mock_which.side_effect = which_side_effect + + mock_run.return_value = MagicMock( + stdout="conda 23.1.0", stderr="", returncode=0 + ) + with pytest.raises(ValueError, match="does not satisfy"): + verify_requirements(["miniforge>=24.0"]) + + +# --- Script generation tests --- + +class TestCondaActivationInScript: + """Test that conda activation appears in the generated script.""" + + @pytest.fixture + def _make_entry_point(self): + def factory(**kwargs): + defaults = dict( + id="test", name="Test", command="python run.py", parameters=[] + ) + defaults.update(kwargs) + return AppEntryPoint(**defaults) + return factory + + def test_script_includes_conda_activation(self, _make_entry_point): + """When conda_env is set, script should contain conda activation lines.""" + import shlex + ep = _make_entry_point(conda_env="myenv") + + # Simulate the script building logic from submit_job + script_parts = ["# preamble"] + if ep.conda_env: + conda_activation = ( + 'eval "$(conda shell.bash hook)"\n' + f'conda activate {shlex.quote(ep.conda_env)}' + ) + script_parts.append(conda_activation) + script_parts.append(ep.command) + full_script = "\n\n".join(script_parts) + + assert 'eval "$(conda shell.bash hook)"' in full_script + assert "conda activate myenv" in full_script + # Activation should come before the command + hook_pos = full_script.index('eval "$(conda shell.bash hook)"') + cmd_pos = full_script.index("python run.py") + assert hook_pos < cmd_pos + + def test_script_omits_conda_when_not_set(self, _make_entry_point): + """When conda_env is None, script should not contain conda activation.""" + ep = _make_entry_point(conda_env=None) + + script_parts = ["# preamble"] + if ep.conda_env: + script_parts.append("conda activate something") + script_parts.append(ep.command) + full_script = "\n\n".join(script_parts) + + assert "conda" not in full_script + + def test_conda_env_path_is_quoted(self, _make_entry_point): + """Absolute paths should be shell-quoted in the script.""" + import shlex + ep = _make_entry_point(conda_env="/opt/conda/envs/my env") + # This would fail validation (spaces in path name, not absolute path forbidden chars) + # but let's test with a valid path containing special-but-allowed chars + ep2 = _make_entry_point(conda_env="/opt/conda/envs/myenv") + + activation = f'conda activate {shlex.quote(ep2.conda_env)}' + assert activation == "conda activate /opt/conda/envs/myenv" From 817ae5aeb85a3de2ead7310625ce1d25da8b04c3 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sat, 28 Feb 2026 19:21:07 -0500 Subject: [PATCH 14/20] added container support --- docs/AuthoringApps.md | 68 ++++++- .../483b2e31b85e_add_container_args_column.py | 26 +++ fileglancer/apps.py | 72 ++++++++ fileglancer/database.py | 8 +- fileglancer/model.py | 52 +++++- fileglancer/server.py | 4 + frontend/src/components/AppJobs.tsx | 4 +- frontend/src/components/AppLaunch.tsx | 14 +- frontend/src/components/JobDetail.tsx | 4 +- .../components/ui/AppsPage/AppLaunchForm.tsx | 90 ++++++++- frontend/src/shared.types.ts | 6 + tests/test_apps.py | 174 +++++++++++++++++- 12 files changed, 506 insertions(+), 16 deletions(-) create mode 100644 fileglancer/alembic/versions/483b2e31b85e_add_container_args_column.py diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index eb1f14ad..b9988fd9 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -67,7 +67,7 @@ requirements: - miniforge ``` -**Supported tools:** `pixi`, `npm`, `maven`, `miniforge` +**Supported tools:** `pixi`, `npm`, `maven`, `miniforge`, `apptainer` **Supported version operators:** `>=`, `<=`, `!=`, `==`, `>`, `<` @@ -90,6 +90,9 @@ Each runnable defines a single command that users can launch. If the manifest ha | `pre_run` | string | no | Shell script to run before the main command (see [Pre/Post-Run Scripts](#prepost-run-scripts)) | | `post_run` | string | no | Shell script to run after the main command (see [Pre/Post-Run Scripts](#prepost-run-scripts)) | | `conda_env` | string | no | Conda environment name or absolute path to activate before running (see [Conda Environments](#conda-environments)) | +| `container` | string | no | Container image URL for Apptainer (see [Containers](#containers-apptainer)) | +| `bind_paths` | list of strings | no | Additional paths to bind-mount into the container (requires `container`) | +| `container_args` | string | no | Default extra arguments for container exec (e.g. `--nv`), overridable at launch time | ### Parameters @@ -287,6 +290,69 @@ conda activate my-analysis-env > conda env create -f environment.yml -n my-tool-env --yes 2>/dev/null || true > ``` +### Containers (Apptainer) + +The `container` field specifies a container image to run the command inside using [Apptainer](https://apptainer.org/) (formerly Singularity). This requires `apptainer` in the `requirements` list. + +The value is a container image URL, typically from a Docker/OCI registry: + +- `ghcr.io/org/image:tag` — GitHub Container Registry +- `docker://ghcr.io/org/image:tag` — explicit Docker protocol prefix (added automatically if absent) + +```yaml +name: Lolcow +requirements: + - apptainer +runnables: + - id: say + name: Cow Say + command: cowsay + container: godlovedc/lolcow + parameters: + - name: Message + type: string + description: What the cow should say + required: true + default: "Hello from Fileglancer!" +``` + +When `container` is set, the generated script: + +1. Creates a SIF cache directory at `$APPTAINER_CACHEDIR` (defaults to `~/.apptainer/cache/fileglancer/`) +2. Pulls the image to a `.sif` file if not already cached +3. Runs the command inside the container via `apptainer exec` + +```bash +# Apptainer container setup +APPTAINER_CACHE_DIR="${APPTAINER_CACHEDIR:-$HOME/.apptainer/cache/fileglancer}" +mkdir -p "$APPTAINER_CACHE_DIR" +SIF_PATH="$APPTAINER_CACHE_DIR/godlovedc_lolcow.sif" +if [ ! -f "$SIF_PATH" ]; then + apptainer pull "$SIF_PATH" 'docker://godlovedc/lolcow' +fi +apptainer exec --bind /home/user/.fileglancer/jobs/1-lolcow-say "$SIF_PATH" \ + cowsay \ + 'Hello from Fileglancer!' +``` + +**Bind mounts** are auto-detected from file and directory parameters. The job's working directory is always bound. Use `bind_paths` to add extra paths: + +```yaml +container: ghcr.io/org/image:tag +bind_paths: + - /shared/reference-data + - /scratch +``` + +**Extra Apptainer arguments** can be set as defaults in the manifest with `container_args`, and overridden by the user at launch time through the UI's Environment tab: + +```yaml +container: ghcr.io/org/cuda-tool:latest +container_args: "--nv" +``` + +> **Important:** `conda_env` and `container` are mutually exclusive — you cannot use both on the same entry point. + ## Command Building When a job is submitted, Fileglancer constructs the full shell command from the runnable's `command` field and the user-provided parameter values using a two-pass approach: diff --git a/fileglancer/alembic/versions/483b2e31b85e_add_container_args_column.py b/fileglancer/alembic/versions/483b2e31b85e_add_container_args_column.py new file mode 100644 index 00000000..3c9c1d7f --- /dev/null +++ b/fileglancer/alembic/versions/483b2e31b85e_add_container_args_column.py @@ -0,0 +1,26 @@ +"""add container and container_args columns + +Revision ID: 483b2e31b85e +Revises: b5e8a3f21c76 +Create Date: 2026-02-28 16:39:39.385514 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '483b2e31b85e' +down_revision = 'b5e8a3f21c76' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column('jobs', sa.Column('container', sa.String(), nullable=True)) + op.add_column('jobs', sa.Column('container_args', sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column('jobs', 'container_args') + op.drop_column('jobs', 'container') diff --git a/fileglancer/apps.py b/fileglancer/apps.py index b6a4c916..6764beea 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -192,6 +192,10 @@ async def fetch_app_manifest(url: str, manifest_path: str = "") -> AppManifest: "version_args": ["conda", "--version"], "version_pattern": r"conda (\S+)", }, + "apptainer": { + "version_args": ["apptainer", "--version"], + "version_pattern": r"apptainer version (\S+)", + }, } _REQ_PATTERN = re.compile(r"^([a-zA-Z][a-zA-Z0-9_-]*)\s*((?:>=|<=|!=|==|>|<)\s*\S+)?$") @@ -564,6 +568,46 @@ def _sanitize_for_path(s: str) -> str: return re.sub(r'[^a-zA-Z0-9._-]', '_', s) +_CONTAINER_SIF_SAFE = re.compile(r'[^a-zA-Z0-9._-]') + + +def _container_sif_name(container_url: str) -> str: + """Derive a safe SIF filename from a container URL.""" + url = container_url.removeprefix("docker://") + return _CONTAINER_SIF_SAFE.sub('_', url) + ".sif" + + +def _build_container_script( + container_url: str, + command: str, + work_dir: str, + bind_paths: list[str], + container_args: Optional[str] = None, +) -> str: + """Build shell script for running a command inside an Apptainer container.""" + sif_name = _container_sif_name(container_url) + docker_url = container_url if container_url.startswith("docker://") else f"docker://{container_url}" + + # Deduplicate and sort bind paths + all_binds = sorted(set([work_dir] + bind_paths)) + bind_flags = " ".join(f"--bind {shlex.quote(p)}" for p in all_binds) + + extra = f" {container_args}" if container_args else "" + + lines = [ + "# Apptainer container setup", + 'APPTAINER_CACHE_DIR="${APPTAINER_CACHEDIR:-$HOME/.apptainer/cache/fileglancer}"', + 'mkdir -p "$APPTAINER_CACHE_DIR"', + f'SIF_PATH="$APPTAINER_CACHE_DIR/{sif_name}"', + 'if [ ! -f "$SIF_PATH" ]; then', + f' apptainer pull "$SIF_PATH" {shlex.quote(docker_url)}', + 'fi', + f'apptainer exec {bind_flags}{extra} "$SIF_PATH" \\', + f' {command}', + ] + return "\n".join(lines) + + def _build_work_dir(job_id: int, app_name: str, entry_point_id: str) -> Path: """Build a working directory path under ~/.fileglancer/jobs/.""" safe_app = _sanitize_for_path(app_name) @@ -583,6 +627,8 @@ async def submit_job( env: Optional[dict] = None, pre_run: Optional[str] = None, post_run: Optional[str] = None, + container: Optional[str] = None, + container_args: Optional[str] = None, ) -> db.JobDB: """Submit a new job to the cluster. @@ -622,6 +668,8 @@ async def submit_job( merged_env.update(env) effective_pre_run = pre_run if pre_run is not None else (entry_point.pre_run or None) effective_post_run = post_run if post_run is not None else (entry_point.post_run or None) + effective_container = container if container is not None else (entry_point.container or None) + effective_container_args = container_args if container_args is not None else (entry_point.container_args or None) # Create DB record first to get job ID for the work directory resources_dict = None @@ -650,6 +698,8 @@ async def submit_job( pre_run=effective_pre_run, post_run=effective_post_run, pull_latest=pull_latest, + container=effective_container, + container_args=effective_container_args, ) job_id = db_job.id @@ -703,6 +753,28 @@ async def submit_job( ) script_parts.append(conda_activation) + # If container is defined, wrap command in apptainer exec + if effective_container: + bind_paths = [] + for param in entry_point.flat_parameters(): + if param.type in ("file", "directory") and param.key in parameters: + path_val = str(parameters[param.key]) + expanded = os.path.expanduser(path_val) + if param.type == "directory": + bind_paths.append(expanded) + else: + bind_paths.append(str(Path(expanded).parent)) + if entry_point.bind_paths: + bind_paths.extend(entry_point.bind_paths) + + command = _build_container_script( + container_url=effective_container, + command=command, + work_dir=str(work_dir), + bind_paths=bind_paths, + container_args=effective_container_args, + ) + if env_lines: script_parts.append(env_lines.rstrip()) if effective_pre_run: diff --git a/fileglancer/database.py b/fileglancer/database.py index 7adfdb63..dc495609 100644 --- a/fileglancer/database.py +++ b/fileglancer/database.py @@ -156,6 +156,8 @@ class JobDB(Base): env = Column(JSON, nullable=True) pre_run = Column(String, nullable=True) post_run = Column(String, nullable=True) + container = Column(String, nullable=True) + container_args = Column(String, nullable=True) pull_latest = Column(Boolean, nullable=False, default=False) created_at = Column(DateTime, nullable=False, default=lambda: datetime.now(UTC)) started_at = Column(DateTime, nullable=True) @@ -831,7 +833,9 @@ def create_job(session: Session, username: str, app_url: str, app_name: str, resources: Optional[Dict] = None, manifest_path: str = "", entry_point_type: str = "job", env: Optional[Dict] = None, pre_run: Optional[str] = None, - post_run: Optional[str] = None, pull_latest: bool = False) -> JobDB: + post_run: Optional[str] = None, pull_latest: bool = False, + container: Optional[str] = None, + container_args: Optional[str] = None) -> JobDB: """Create a new job record""" now = datetime.now(UTC) job = JobDB( @@ -847,6 +851,8 @@ def create_job(session: Session, username: str, app_url: str, app_name: str, env=env, pre_run=pre_run, post_run=post_run, + container=container, + container_args=container_args, pull_latest=pull_latest, status="PENDING", created_at=now diff --git a/fileglancer/model.py b/fileglancer/model.py index da74d2e3..2378b7fa 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -395,6 +395,18 @@ class AppEntryPoint(BaseModel): description="Conda environment name or path to activate before running", default=None, ) + container: Optional[str] = Field( + description="Container image URL for Apptainer (e.g. 'ghcr.io/org/image:tag')", + default=None, + ) + bind_paths: Optional[List[str]] = Field( + description="Additional paths to bind-mount into the container", + default=None, + ) + container_args: Optional[str] = Field( + description="Default extra arguments for container exec (e.g. '--nv')", + default=None, + ) @field_validator("conda_env") @classmethod @@ -415,6 +427,25 @@ def validate_conda_env(cls, v): ) return v + @field_validator("container") + @classmethod + def validate_container(cls, v): + if v is None: + return v + if _SHELL_METACHAR_PATTERN.search(v): + raise ValueError(f"container URL contains forbidden characters: {v!r}") + return v + + @field_validator("bind_paths") + @classmethod + def validate_bind_paths(cls, v): + if v is None: + return v + for p in v: + if _SHELL_METACHAR_PATTERN.search(p): + raise ValueError(f"bind_paths entry contains forbidden characters: {p!r}") + return v + def flat_parameters(self) -> List[AppParameter]: """Return a flat list of all parameters, traversing sections.""" result = [] @@ -443,9 +474,18 @@ def generate_parameter_keys(self): keys_seen[param.key] = param.name return self + @model_validator(mode='after') + def check_conda_container_exclusive(self): + if self.conda_env and self.container: + raise ValueError("conda_env and container are mutually exclusive — use one or the other") + if self.bind_paths and not self.container: + raise ValueError("bind_paths requires container to be set") + return self -SUPPORTED_TOOLS = {"pixi", "npm", "maven", "miniforge"} +SUPPORTED_TOOLS = {"pixi", "npm", "maven", "miniforge", "apptainer"} + +_SHELL_METACHAR_PATTERN = re.compile(r'[;&|`$(){}!<>\n\r]') _CONDA_ENV_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9_.-]+$') _CONDA_ENV_PATH_FORBIDDEN = re.compile(r'[;&|`$(){}!<>\n\r]') @@ -525,6 +565,8 @@ class Job(BaseModel): env: Optional[Dict[str, str]] = Field(description="Environment variables used for the job", default=None) pre_run: Optional[str] = Field(description="Script run before the main command", default=None) post_run: Optional[str] = Field(description="Script run after the main command", default=None) + container: Optional[str] = Field(description="Container image URL used for this job", default=None) + container_args: Optional[str] = Field(description="Extra arguments for container exec (e.g. '--nv' for GPU)", default=None) pull_latest: bool = Field(description="Whether pull latest was enabled", default=False) cluster_job_id: Optional[str] = Field(description="Cluster-assigned job ID", default=None) service_url: Optional[str] = Field(description="URL of the running service (for service-type jobs)", default=None) @@ -549,6 +591,14 @@ class JobSubmitRequest(BaseModel): env: Optional[Dict[str, str]] = Field(description="Environment variables to export", default=None) pre_run: Optional[str] = Field(description="Script to run before the main command", default=None) post_run: Optional[str] = Field(description="Script to run after the main command", default=None) + container: Optional[str] = Field( + description="Container image URL override (defaults to manifest value)", + default=None, + ) + container_args: Optional[str] = Field( + description="Extra arguments for container exec (e.g. '--nv' for GPU)", + default=None, + ) class PathValidationRequest(BaseModel): diff --git a/fileglancer/server.py b/fileglancer/server.py index c6418aa9..c46d728b 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1662,6 +1662,8 @@ async def submit_job(body: JobSubmitRequest, env=body.env, pre_run=body.pre_run, post_run=body.post_run, + container=body.container, + container_args=body.container_args, ) return _convert_job(db_job) except ValueError as e: @@ -1757,6 +1759,8 @@ def _convert_job(db_job: db.JobDB, include_files: bool = False) -> Job: env=db_job.env, pre_run=db_job.pre_run, post_run=db_job.post_run, + container=getattr(db_job, 'container', None), + container_args=getattr(db_job, 'container_args', None), pull_latest=db_job.pull_latest, cluster_job_id=db_job.cluster_job_id, service_url=apps_module.get_service_url(db_job), diff --git a/frontend/src/components/AppJobs.tsx b/frontend/src/components/AppJobs.tsx index 0410ab55..f458408d 100644 --- a/frontend/src/components/AppJobs.tsx +++ b/frontend/src/components/AppJobs.tsx @@ -40,7 +40,9 @@ export default function AppJobs() { env: job.env, pre_run: job.pre_run, post_run: job.post_run, - pull_latest: job.pull_latest + pull_latest: job.pull_latest, + container: job.container, + container_args: job.container_args } }); }; diff --git a/frontend/src/components/AppLaunch.tsx b/frontend/src/components/AppLaunch.tsx index 222e8d37..40888a8e 100644 --- a/frontend/src/components/AppLaunch.tsx +++ b/frontend/src/components/AppLaunch.tsx @@ -52,6 +52,8 @@ export default function AppLaunch() { pre_run?: string; post_run?: string; pull_latest?: boolean; + container?: string; + container_args?: string; } | null) : null; const relaunchParameters = relaunchState?.parameters; @@ -66,6 +68,8 @@ export default function AppLaunch() { const relaunchPreRun = relaunchState?.pre_run; const relaunchPostRun = relaunchState?.post_run; const relaunchPullLatest = relaunchState?.pull_latest; + const relaunchContainer = relaunchState?.container; + const relaunchContainerArgs = relaunchState?.container_args; // Check if app is in user's library const isInstalled = appsQuery.data?.some( @@ -107,7 +111,9 @@ export default function AppLaunch() { pullLatest?: boolean, env?: Record, preRun?: string, - postRun?: string + postRun?: string, + container?: string, + containerArgs?: string ) => { if (!selectedEntryPoint) { return; @@ -123,7 +129,9 @@ export default function AppLaunch() { pull_latest: pullLatest, env, pre_run: preRun, - post_run: postRun + post_run: postRun, + container, + container_args: containerArgs }); const isService = selectedEntryPoint.type === 'service'; toast.success(isService ? 'Service started' : 'Job submitted'); @@ -209,6 +217,8 @@ export default function AppLaunch() { ) : manifest && selectedEntryPoint ? ( , preRun?: string, - postRun?: string + postRun?: string, + container?: string, + containerArgs?: string ) => Promise; readonly submitting: boolean; readonly initialValues?: Record; @@ -43,6 +45,8 @@ interface AppLaunchFormProps { readonly initialPreRun?: string; readonly initialPostRun?: string; readonly initialPullLatest?: boolean; + readonly initialContainer?: string; + readonly initialContainerArgs?: string; } type EnvVar = { key: string; value: string }; @@ -478,7 +482,12 @@ function EnvironmentTabContent({ postRun, setPostRun, openEnvSections, - setOpenEnvSections + setOpenEnvSections, + entryPoint, + containerImage, + setContainerImage, + containerArgs, + setContainerArgs }: { readonly envVars: EnvVar[]; readonly setEnvVars: Dispatch>; @@ -488,7 +497,15 @@ function EnvironmentTabContent({ readonly setPostRun: Dispatch>; readonly openEnvSections: string[]; readonly setOpenEnvSections: Dispatch>; + readonly entryPoint: AppEntryPoint; + readonly containerImage: string; + readonly setContainerImage: Dispatch>; + readonly containerArgs: string; + readonly setContainerArgs: Dispatch>; }) { + const inputClass = + 'w-full p-2 text-foreground border rounded-sm focus:outline-none bg-background border-primary-light focus:border-primary'; + return ( + + {entryPoint.container ? ( + + +
Container
+ +
+ +
+
+ + setContainerImage(e.target.value)} + placeholder="e.g. ghcr.io/org/image:tag" + type="text" + value={containerImage} + /> +
+
+ + + Additional flags passed to apptainer exec + + setContainerArgs(e.target.value)} + placeholder="e.g. --nv" + type="text" + value={containerArgs} + /> +
+
+
+
+ ) : null}
); } @@ -596,7 +657,9 @@ export default function AppLaunchForm({ initialEnv, initialPreRun, initialPostRun, - initialPullLatest + initialPullLatest, + initialContainer, + initialContainerArgs }: AppLaunchFormProps) { const { defaultExtraArgs } = usePreferencesContext(); const clusterDefaultsQuery = useClusterDefaultsQuery(); @@ -653,9 +716,15 @@ export default function AppLaunchForm({ const [postRun, setPostRun] = useState( initialPostRun ?? entryPoint.post_run ?? '' ); - const [openEnvSections, setOpenEnvSections] = useState([ - 'environment' - ]); + const [containerImage, setContainerImage] = useState( + initialContainer ?? entryPoint.container ?? '' + ); + const [containerArgs, setContainerArgs] = useState( + initialContainerArgs ?? entryPoint.container_args ?? '' + ); + const [openEnvSections, setOpenEnvSections] = useState( + entryPoint.container ? ['environment', 'apptainer'] : ['environment'] + ); const [openClusterSections, setOpenClusterSections] = useState([ 'resources', 'submitOptions' @@ -807,7 +876,9 @@ export default function AppLaunchForm({ pullLatest, hasEnv ? envRecord : undefined, preRun.trim() || undefined, - postRun.trim() || undefined + postRun.trim() || undefined, + containerImage.trim() || undefined, + containerArgs.trim() || undefined ); }; @@ -935,10 +1006,15 @@ export default function AppLaunchForm({
; pre_run?: string; post_run?: string; + container?: string; + container_args?: string; pull_latest: boolean; cluster_job_id?: string; service_url?: string; @@ -180,6 +184,8 @@ type JobSubmitRequest = { env?: Record; pre_run?: string; post_run?: string; + container?: string; + container_args?: string; }; export type { diff --git a/tests/test_apps.py b/tests/test_apps.py index 23776248..a7fe1c5a 100644 --- a/tests/test_apps.py +++ b/tests/test_apps.py @@ -1,4 +1,4 @@ -"""Tests for apps module: miniforge requirement and conda_env support.""" +"""Tests for apps module: miniforge/apptainer requirements, conda_env, and container support.""" import subprocess from unittest.mock import patch, MagicMock @@ -7,7 +7,12 @@ from pydantic import ValidationError from fileglancer.model import SUPPORTED_TOOLS, AppEntryPoint -from fileglancer.apps import _TOOL_REGISTRY, verify_requirements +from fileglancer.apps import ( + _TOOL_REGISTRY, + verify_requirements, + _container_sif_name, + _build_container_script, +) # --- Model tests --- @@ -186,3 +191,168 @@ def test_conda_env_path_is_quoted(self, _make_entry_point): activation = f'conda activate {shlex.quote(ep2.conda_env)}' assert activation == "conda activate /opt/conda/envs/myenv" + + +# --- Apptainer / Container tests --- + +class TestApptainerRequirement: + def test_apptainer_in_supported_tools(self): + assert "apptainer" in SUPPORTED_TOOLS + + def test_apptainer_in_tool_registry(self): + assert "apptainer" in _TOOL_REGISTRY + entry = _TOOL_REGISTRY["apptainer"] + assert entry["version_args"] == ["apptainer", "--version"] + assert entry["version_pattern"] == r"apptainer version (\S+)" + + +class TestContainerValidation: + def test_valid_container_url(self): + ep = AppEntryPoint( + id="t", name="T", command="echo", + container="ghcr.io/org/image:tag" + ) + assert ep.container == "ghcr.io/org/image:tag" + + def test_valid_docker_prefix(self): + ep = AppEntryPoint( + id="t", name="T", command="echo", + container="docker://ghcr.io/org/image:1.0" + ) + assert ep.container == "docker://ghcr.io/org/image:1.0" + + def test_none_is_allowed(self): + ep = AppEntryPoint(id="t", name="T", command="echo", container=None) + assert ep.container is None + + def test_default_is_none(self): + ep = AppEntryPoint(id="t", name="T", command="echo") + assert ep.container is None + + def test_rejects_shell_metacharacters(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint( + id="t", name="T", command="echo", + container="ghcr.io/org/image;rm -rf /" + ) + + def test_rejects_backtick(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint( + id="t", name="T", command="echo", + container="ghcr.io/`whoami`/image:tag" + ) + + def test_mutual_exclusion_with_conda(self): + with pytest.raises(ValidationError, match="mutually exclusive"): + AppEntryPoint( + id="t", name="T", command="echo", + conda_env="myenv", + container="ghcr.io/org/image:tag" + ) + + def test_bind_paths_requires_container(self): + with pytest.raises(ValidationError, match="bind_paths requires container"): + AppEntryPoint( + id="t", name="T", command="echo", + bind_paths=["/data"] + ) + + def test_bind_paths_with_container(self): + ep = AppEntryPoint( + id="t", name="T", command="echo", + container="ghcr.io/org/image:tag", + bind_paths=["/data", "/scratch"] + ) + assert ep.bind_paths == ["/data", "/scratch"] + + def test_bind_paths_rejects_metacharacters(self): + with pytest.raises(ValidationError, match="forbidden characters"): + AppEntryPoint( + id="t", name="T", command="echo", + container="ghcr.io/org/image:tag", + bind_paths=["/data;rm -rf /"] + ) + + +class TestContainerSifName: + def test_simple_url(self): + assert _container_sif_name("ghcr.io/org/image:1.0") == "ghcr.io_org_image_1.0.sif" + + def test_docker_prefix_stripped(self): + assert _container_sif_name("docker://ghcr.io/org/image:tag") == "ghcr.io_org_image_tag.sif" + + def test_nested_path(self): + result = _container_sif_name("godlovedc/lolcow") + assert result == "godlovedc_lolcow.sif" + + def test_no_tag(self): + result = _container_sif_name("ghcr.io/org/image") + assert result == "ghcr.io_org_image.sif" + + +class TestContainerScriptGeneration: + def test_basic_script(self): + script = _build_container_script( + container_url="ghcr.io/org/image:1.0", + command="python run.py", + work_dir="/home/user/.fileglancer/jobs/1-test-run", + bind_paths=[], + ) + assert "apptainer pull" in script + assert "apptainer exec" in script + assert "docker://ghcr.io/org/image:1.0" in script + assert "ghcr.io_org_image_1.0.sif" in script + assert "python run.py" in script + + def test_bind_mounts_included(self): + script = _build_container_script( + container_url="ghcr.io/org/image:1.0", + command="echo hello", + work_dir="/work", + bind_paths=["/data/input", "/data/output"], + ) + assert "--bind /data/input" in script + assert "--bind /data/output" in script + assert "--bind /work" in script + + def test_bind_mounts_deduplicated(self): + script = _build_container_script( + container_url="ghcr.io/org/image:1.0", + command="echo hello", + work_dir="/work", + bind_paths=["/work", "/data", "/data"], + ) + # /work should only appear once in bind flags + assert script.count("--bind /work") == 1 + assert script.count("--bind /data") == 1 + + def test_extra_args(self): + script = _build_container_script( + container_url="ghcr.io/org/image:1.0", + command="python run.py", + work_dir="/work", + bind_paths=[], + container_args="--nv", + ) + assert "--nv" in script + + def test_pull_conditional(self): + script = _build_container_script( + container_url="ghcr.io/org/image:1.0", + command="echo", + work_dir="/work", + bind_paths=[], + ) + assert 'if [ ! -f "$SIF_PATH" ]' in script + + def test_docker_prefix_not_doubled(self): + script = _build_container_script( + container_url="docker://ghcr.io/org/image:1.0", + command="echo", + work_dir="/work", + bind_paths=[], + ) + # Should not have docker://docker:// + assert "docker://docker://" not in script + assert "docker://ghcr.io/org/image:1.0" in script From d495b78c20f9adaa6acf1a4bd4fbbf8dffd81632 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sat, 28 Feb 2026 19:26:25 -0500 Subject: [PATCH 15/20] move apptainer cache and add preference --- docs/AuthoringApps.md | 4 +- fileglancer/apps.py | 13 +- .../ui/PreferencesPage/JobOptions.tsx | 120 ++++++++++++------ frontend/src/contexts/PreferencesContext.tsx | 18 +++ frontend/src/queries/preferencesQueries.ts | 5 +- 5 files changed, 120 insertions(+), 40 deletions(-) diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index b9988fd9..575bac8e 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -318,13 +318,13 @@ runnables: When `container` is set, the generated script: -1. Creates a SIF cache directory at `$APPTAINER_CACHEDIR` (defaults to `~/.apptainer/cache/fileglancer/`) +1. Creates a SIF cache directory (defaults to `~/.fileglancer/apptainer_cache/`, configurable in Preferences) 2. Pulls the image to a `.sif` file if not already cached 3. Runs the command inside the container via `apptainer exec` ```bash # Apptainer container setup -APPTAINER_CACHE_DIR="${APPTAINER_CACHEDIR:-$HOME/.apptainer/cache/fileglancer}" +APPTAINER_CACHE_DIR=$HOME/.fileglancer/apptainer_cache mkdir -p "$APPTAINER_CACHE_DIR" SIF_PATH="$APPTAINER_CACHE_DIR/godlovedc_lolcow.sif" if [ ! -f "$SIF_PATH" ]; then diff --git a/fileglancer/apps.py b/fileglancer/apps.py index 6764beea..ba8d3f50 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -577,12 +577,16 @@ def _container_sif_name(container_url: str) -> str: return _CONTAINER_SIF_SAFE.sub('_', url) + ".sif" +_DEFAULT_CONTAINER_CACHE_DIR = "$HOME/.fileglancer/apptainer_cache" + + def _build_container_script( container_url: str, command: str, work_dir: str, bind_paths: list[str], container_args: Optional[str] = None, + cache_dir: Optional[str] = None, ) -> str: """Build shell script for running a command inside an Apptainer container.""" sif_name = _container_sif_name(container_url) @@ -594,9 +598,11 @@ def _build_container_script( extra = f" {container_args}" if container_args else "" + resolved_dir = shlex.quote(cache_dir) if cache_dir else _DEFAULT_CONTAINER_CACHE_DIR + lines = [ "# Apptainer container setup", - 'APPTAINER_CACHE_DIR="${APPTAINER_CACHEDIR:-$HOME/.apptainer/cache/fileglancer}"', + f'APPTAINER_CACHE_DIR={resolved_dir}', 'mkdir -p "$APPTAINER_CACHE_DIR"', f'SIF_PATH="$APPTAINER_CACHE_DIR/{sif_name}"', 'if [ ! -f "$SIF_PATH" ]; then', @@ -683,6 +689,10 @@ async def submit_job( } with db.get_db_session(settings.db_url) as session: + # Read user's container cache dir preference + cache_dir_pref = db.get_user_preference(session, username, "apptainerCacheDir") + container_cache_dir = cache_dir_pref.get("value") if cache_dir_pref else None + db_job = db.create_job( session=session, username=username, @@ -773,6 +783,7 @@ async def submit_job( work_dir=str(work_dir), bind_paths=bind_paths, container_args=effective_container_args, + cache_dir=container_cache_dir, ) if env_lines: diff --git a/frontend/src/components/ui/PreferencesPage/JobOptions.tsx b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx index f6317eff..ac2575d6 100644 --- a/frontend/src/components/ui/PreferencesPage/JobOptions.tsx +++ b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx @@ -6,18 +6,25 @@ import toast from 'react-hot-toast'; import { usePreferencesContext } from '@/contexts/PreferencesContext'; export default function JobOptions() { - const { defaultExtraArgs, updateDefaultExtraArgs } = usePreferencesContext(); + const { + defaultExtraArgs, + updateDefaultExtraArgs, + apptainerCacheDir, + updateApptainerCacheDir + } = usePreferencesContext(); - const [localValue, setLocalValue] = useState(defaultExtraArgs); - const [saving, setSaving] = useState(false); + const [localExtraArgs, setLocalExtraArgs] = useState(defaultExtraArgs); + const [savingExtraArgs, setSavingExtraArgs] = useState(false); + const isExtraArgsDirty = localExtraArgs !== defaultExtraArgs; - // Track whether the local value differs from the saved preference - const isDirty = localValue !== defaultExtraArgs; + const [localCacheDir, setLocalCacheDir] = useState(apptainerCacheDir); + const [savingCacheDir, setSavingCacheDir] = useState(false); + const isCacheDirDirty = localCacheDir !== apptainerCacheDir; - const handleSave = async () => { - setSaving(true); - const result = await updateDefaultExtraArgs(localValue.trim()); - setSaving(false); + const handleSaveExtraArgs = async () => { + setSavingExtraArgs(true); + const result = await updateDefaultExtraArgs(localExtraArgs.trim()); + setSavingExtraArgs(false); if (result.success) { toast.success('Default extra arguments saved'); } else { @@ -25,36 +32,77 @@ export default function JobOptions() { } }; + const handleSaveCacheDir = async () => { + setSavingCacheDir(true); + const result = await updateApptainerCacheDir(localCacheDir.trim()); + setSavingCacheDir(false); + if (result.success) { + toast.success('Container cache directory saved'); + } else { + toast.error(result.error); + } + }; + return (
Jobs -
- - - Additional CLI arguments appended to every job submit command. Can be - overridden per job on the Cluster tab. - - setLocalValue(e.target.value)} - placeholder="e.g. -P your_project" - type="text" - value={localValue} - /> - +
+
+ + + Additional CLI arguments appended to every job submit command. Can + be overridden per job on the Cluster tab. + + setLocalExtraArgs(e.target.value)} + placeholder="e.g. -P your_project" + type="text" + value={localExtraArgs} + /> + +
+ +
+ + + Directory where Apptainer SIF images are cached. Defaults to ~/.fileglancer/apptainer_cache if not set. + + setLocalCacheDir(e.target.value)} + placeholder="~/.fileglancer/apptainer_cache" + type="text" + value={localCacheDir} + /> + +
); diff --git a/frontend/src/contexts/PreferencesContext.tsx b/frontend/src/contexts/PreferencesContext.tsx index 3c75c28d..e1c4e8d9 100644 --- a/frontend/src/contexts/PreferencesContext.tsx +++ b/frontend/src/contexts/PreferencesContext.tsx @@ -54,6 +54,7 @@ type PreferencesContextType = { isFilteredByGroups: boolean; showTutorial: boolean; defaultExtraArgs: string; + apptainerCacheDir: string; // Favorites zoneFavorites: Zone[]; @@ -82,6 +83,7 @@ type PreferencesContextType = { toggleFilterByGroups: () => Promise>; toggleShowTutorial: () => Promise>; updateDefaultExtraArgs: (args: string) => Promise>; + updateApptainerCacheDir: (dir: string) => Promise>; handleFavoriteChange: ( item: Zone | FileSharePath | FolderFavorite, type: 'zone' | 'fileSharePath' | 'folder' @@ -258,6 +260,20 @@ export const PreferencesProvider = ({ } }; + const updateApptainerCacheDir = async ( + dir: string + ): Promise> => { + try { + await updatePreferenceMutation.mutateAsync({ + key: 'apptainerCacheDir', + value: dir + }); + return createSuccess(undefined); + } catch (error) { + return handleError(error); + } + }; + function updatePreferenceList( key: string, itemToUpdate: T, @@ -517,6 +533,7 @@ export const PreferencesProvider = ({ isFilteredByGroups: preferencesQuery.data?.isFilteredByGroups ?? true, showTutorial: preferencesQuery.data?.showTutorial ?? true, defaultExtraArgs: preferencesQuery.data?.defaultExtraArgs || '', + apptainerCacheDir: preferencesQuery.data?.apptainerCacheDir || '', // Favorites zoneFavorites: preferencesQuery.data?.zoneFavorites || [], @@ -544,6 +561,7 @@ export const PreferencesProvider = ({ toggleFilterByGroups, toggleShowTutorial, updateDefaultExtraArgs, + updateApptainerCacheDir, handleFavoriteChange, handleContextMenuFavorite }; diff --git a/frontend/src/queries/preferencesQueries.ts b/frontend/src/queries/preferencesQueries.ts index a1aa9bec..57f50815 100644 --- a/frontend/src/queries/preferencesQueries.ts +++ b/frontend/src/queries/preferencesQueries.ts @@ -39,6 +39,7 @@ type PreferencesApiResponse = { isFilteredByGroups?: { value: boolean }; showTutorial?: { value: boolean }; defaultExtraArgs?: { value: string }; + apptainerCacheDir?: { value: string }; zone?: { value: ZonePreference[] }; fileSharePath?: { value: FileSharePathPreference[] }; folder?: { value: FolderPreference[] }; @@ -70,6 +71,7 @@ export type PreferencesQueryData = { isFilteredByGroups: boolean; showTutorial: boolean; defaultExtraArgs: string; + apptainerCacheDir: string; }; /** @@ -235,7 +237,8 @@ const createTransformPreferences = ( rawData.useLegacyMultichannelApproach?.value || false, isFilteredByGroups: rawData.isFilteredByGroups?.value ?? true, showTutorial: rawData.showTutorial?.value ?? true, - defaultExtraArgs: rawData.defaultExtraArgs?.value || '' + defaultExtraArgs: rawData.defaultExtraArgs?.value || '', + apptainerCacheDir: rawData.apptainerCacheDir?.value || '' }; }; }; From 8d8e6de19b413c723afb26d0350b6a8158697a1e Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sun, 1 Mar 2026 06:51:39 -0500 Subject: [PATCH 16/20] added nextflow to supported tools --- docs/AuthoringApps.md | 2 +- fileglancer/apps.py | 4 ++++ fileglancer/model.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index 575bac8e..95894346 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -67,7 +67,7 @@ requirements: - miniforge ``` -**Supported tools:** `pixi`, `npm`, `maven`, `miniforge`, `apptainer` +**Supported tools:** `pixi`, `npm`, `maven`, `miniforge`, `apptainer`, `nextflow` **Supported version operators:** `>=`, `<=`, `!=`, `==`, `>`, `<` diff --git a/fileglancer/apps.py b/fileglancer/apps.py index ba8d3f50..3f920906 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -196,6 +196,10 @@ async def fetch_app_manifest(url: str, manifest_path: str = "") -> AppManifest: "version_args": ["apptainer", "--version"], "version_pattern": r"apptainer version (\S+)", }, + "nextflow": { + "version_args": ["nextflow", "-version"], + "version_pattern": r"version (\S+)", + }, } _REQ_PATTERN = re.compile(r"^([a-zA-Z][a-zA-Z0-9_-]*)\s*((?:>=|<=|!=|==|>|<)\s*\S+)?$") diff --git a/fileglancer/model.py b/fileglancer/model.py index 2378b7fa..0e2037d7 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -483,7 +483,7 @@ def check_conda_container_exclusive(self): return self -SUPPORTED_TOOLS = {"pixi", "npm", "maven", "miniforge", "apptainer"} +SUPPORTED_TOOLS = {"pixi", "npm", "maven", "miniforge", "apptainer", "nextflow"} _SHELL_METACHAR_PATTERN = re.compile(r'[;&|`$(){}!<>\n\r]') _CONDA_ENV_NAME_PATTERN = re.compile(r'^[a-zA-Z0-9_.-]+$') From 886fcf4e3ad7c37e4d6ccc0c5aa657c619e9aef7 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sun, 1 Mar 2026 07:19:21 -0500 Subject: [PATCH 17/20] extra paths, fixed updates --- docs/AuthoringApps.md | 27 +++++++++++++++++++++++++++ docs/config.yaml.template | 3 +++ fileglancer/apps.py | 19 +++++++++++++++++-- fileglancer/model.py | 1 + fileglancer/server.py | 26 ++++++++++++++++++++++---- fileglancer/settings.py | 1 + frontend/src/queries/appsQueries.ts | 4 ++-- frontend/src/shared.types.ts | 1 + tests/test_apps.py | 6 +++--- 9 files changed, 77 insertions(+), 11 deletions(-) diff --git a/docs/AuthoringApps.md b/docs/AuthoringApps.md index 95894346..72a1f7d5 100644 --- a/docs/AuthoringApps.md +++ b/docs/AuthoringApps.md @@ -529,6 +529,33 @@ Fileglancer exports the following environment variables in every job script: These are available to `pre_run` scripts, the main command, and `post_run` scripts. +## Server Configuration for Apps + +Some aspects of app execution are controlled by the Fileglancer server's `config.yaml`, not by individual app manifests. These settings are managed by the system administrator. + +### Extra Paths (`extra_paths`) + +The `extra_paths` cluster setting lets administrators add directories to `$PATH` for all job scripts. This is useful for making tools like `nextflow`, `pixi`, or `apptainer` available without requiring users to configure their own environments. + +```yaml +cluster: + extra_paths: + - /opt/nextflow/bin + - /opt/pixi/bin + - /usr/local/apptainer/bin +``` + +These paths are: + +1. **Appended to `$PATH` in every generated job script** — the user's own `$PATH` entries take precedence +2. **Used when verifying tool requirements** — so `requirements: [nextflow]` can find `/opt/nextflow/bin/nextflow` even if it's not on the server process's default `$PATH` + +### Container Cache Directory + +By default, Apptainer container images (SIF files) are cached at `~/.fileglancer/apptainer_cache/`. Users can override this per-user in the Preferences page under "Container cache directory". + +See the [config.yaml.template](config.yaml.template) for all available cluster settings. + ## Full Example ```yaml diff --git a/docs/config.yaml.template b/docs/config.yaml.template index 9b9fe167..f943ed09 100644 --- a/docs/config.yaml.template +++ b/docs/config.yaml.template @@ -149,3 +149,6 @@ cluster: # - "your_project" # script_prologue: # Commands to run before each job # - "module load java/11" +# extra_paths: # Paths appended to $PATH in every job script +# - /opt/nextflow/bin # and used when verifying tool requirements +# - /opt/pixi/bin diff --git a/fileglancer/apps.py b/fileglancer/apps.py index 3f920906..a594367e 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -205,6 +205,13 @@ async def fetch_app_manifest(url: str, manifest_path: str = "") -> AppManifest: _REQ_PATTERN = re.compile(r"^([a-zA-Z][a-zA-Z0-9_-]*)\s*((?:>=|<=|!=|==|>|<)\s*\S+)?$") +def _augmented_path(extra_paths: list[str]) -> str: + """Build a PATH string with extra_paths appended (user's PATH takes precedence).""" + if not extra_paths: + return os.environ.get("PATH", "") + return os.environ.get("PATH", "") + os.pathsep + os.pathsep.join(extra_paths) + + def verify_requirements(requirements: list[str]): """Verify that all required tools are available and meet version constraints. @@ -213,6 +220,10 @@ def verify_requirements(requirements: list[str]): if not requirements: return + settings = get_settings() + search_path = _augmented_path(settings.cluster.extra_paths) + env = {**os.environ, "PATH": search_path} if settings.cluster.extra_paths else None + errors = [] for req in requirements: @@ -225,11 +236,11 @@ def verify_requirements(requirements: list[str]): version_spec = match.group(2) # Check tool exists on PATH - if shutil.which(tool) is None: + if shutil.which(tool, path=search_path) is None: # For maven, the binary is 'mvn' not 'maven' registry_entry = _TOOL_REGISTRY.get(tool) binary = registry_entry["version_args"][0] if registry_entry else tool - if binary != tool and shutil.which(binary) is not None: + if binary != tool and shutil.which(binary, path=search_path) is not None: pass # binary found under alternate name else: errors.append(f"Required tool '{tool}' is not installed or not on PATH") @@ -245,6 +256,7 @@ def verify_requirements(requirements: list[str]): result = subprocess.run( registry_entry["version_args"], capture_output=True, text=True, timeout=10, + env=env, ) output = result.stdout.strip() or result.stderr.strip() ver_match = re.search(registry_entry["version_pattern"], output) @@ -754,6 +766,9 @@ async def submit_job( "unset PIXI_PROJECT_MANIFEST", f"export FG_WORK_DIR={shlex.quote(str(work_dir))}", ] + if settings.cluster.extra_paths: + path_suffix = os.pathsep.join(shlex.quote(p) for p in settings.cluster.extra_paths) + preamble_lines.append(f"export PATH=$PATH:{path_suffix}") if entry_point.type == "service": preamble_lines.append('export SERVICE_URL_PATH="$FG_WORK_DIR/service_url"') preamble_lines.append(f'cd "$FG_WORK_DIR/{cd_suffix}"') diff --git a/fileglancer/model.py b/fileglancer/model.py index 0e2037d7..56712f40 100644 --- a/fileglancer/model.py +++ b/fileglancer/model.py @@ -522,6 +522,7 @@ class UserApp(BaseModel): name: str = Field(description="App name from manifest") description: Optional[str] = Field(description="App description from manifest", default=None) added_at: datetime = Field(description="When the app was added") + updated_at: Optional[datetime] = Field(description="When the app was last updated", default=None) manifest: Optional[AppManifest] = Field(description="Cached manifest data", default=None) diff --git a/fileglancer/server.py b/fileglancer/server.py index c46d728b..acc5f107 100644 --- a/fileglancer/server.py +++ b/fileglancer/server.py @@ -1492,6 +1492,7 @@ async def get_user_apps(username: str = Depends(get_current_user)): name=app_entry.get("name", "Unknown"), description=app_entry.get("description"), added_at=app_entry.get("added_at", datetime.now(UTC).isoformat()), + updated_at=app_entry.get("updated_at"), ) # Try to fetch manifest from local clone try: @@ -1590,30 +1591,47 @@ async def remove_user_app(url: str = Query(..., description="URL of the app to r return {"message": "App removed"} - @app.post("/api/apps/update", response_model=AppManifest, + @app.post("/api/apps/update", response_model=UserApp, description="Pull latest code and re-read the manifest for an app") async def update_user_app(body: ManifestFetchRequest, username: str = Depends(get_current_user)): try: await apps_module._ensure_repo_cache(body.url, pull=True) + except Exception as e: + raise HTTPException(status_code=400, detail=f"Failed to pull latest code: {str(e)}") + + try: manifest = await apps_module.fetch_app_manifest(body.url, body.manifest_path) except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) except Exception as e: - raise HTTPException(status_code=400, detail=f"Failed to update app: {str(e)}") + raise HTTPException(status_code=400, detail=f"Failed to read manifest after update: {str(e)}") - # Update stored name/description from refreshed manifest + now = datetime.now(UTC) + + # Update stored name/description/updated_at from refreshed manifest with db.get_db_session(settings.db_url) as session: pref = db.get_user_preference(session, username, "apps") app_list = pref.get("apps", []) if pref else [] + added_at = now # fallback for entry in app_list: if entry["url"] == body.url and entry.get("manifest_path", "") == body.manifest_path: entry["name"] = manifest.name entry["description"] = manifest.description + entry["updated_at"] = now.isoformat() + added_at = entry.get("added_at", now.isoformat()) break db.set_user_preference(session, username, "apps", {"apps": app_list}) - return manifest + return UserApp( + url=body.url, + manifest_path=body.manifest_path, + name=manifest.name, + description=manifest.description, + added_at=added_at, + updated_at=now, + manifest=manifest, + ) @app.post("/api/apps/validate-paths", response_model=PathValidationResponse, description="Validate file/directory paths for app parameters") diff --git a/fileglancer/settings.py b/fileglancer/settings.py index 149a42d3..29c1ee16 100644 --- a/fileglancer/settings.py +++ b/fileglancer/settings.py @@ -32,6 +32,7 @@ class ClusterSettings(BaseModel): completed_retention_minutes: float = 10.0 command_timeout: float = 100.0 suppress_job_email: bool = True + extra_paths: List[str] = [] class Settings(BaseSettings): diff --git a/frontend/src/queries/appsQueries.ts b/frontend/src/queries/appsQueries.ts index 9bf4ec11..4c167d22 100644 --- a/frontend/src/queries/appsQueries.ts +++ b/frontend/src/queries/appsQueries.ts @@ -101,7 +101,7 @@ export async function validatePaths( } export function useUpdateAppMutation(): UseMutationResult< - AppManifest, + UserApp, Error, { url: string; manifest_path: string } > { @@ -122,7 +122,7 @@ export function useUpdateAppMutation(): UseMutationResult< if (!response.ok) { throwResponseNotOkError(response, data); } - return data as AppManifest; + return data as UserApp; }, onSuccess: () => { queryClient.invalidateQueries({ queryKey: appsQueryKeys.all }); diff --git a/frontend/src/shared.types.ts b/frontend/src/shared.types.ts index a923c084..36f181ff 100644 --- a/frontend/src/shared.types.ts +++ b/frontend/src/shared.types.ts @@ -137,6 +137,7 @@ type UserApp = { name: string; description?: string; added_at: string; + updated_at?: string; manifest?: AppManifest; }; diff --git a/tests/test_apps.py b/tests/test_apps.py index a7fe1c5a..89c2184d 100644 --- a/tests/test_apps.py +++ b/tests/test_apps.py @@ -80,7 +80,7 @@ class TestVerifyRequirementsMiniforge: @patch("fileglancer.apps.shutil.which") def test_miniforge_found_via_conda(self, mock_which): """miniforge binary doesn't exist, but conda does — should pass.""" - def which_side_effect(name): + def which_side_effect(name, **kwargs): if name == "miniforge": return None if name == "conda": @@ -100,7 +100,7 @@ def test_miniforge_not_found(self, mock_which): @patch("fileglancer.apps.subprocess.run") @patch("fileglancer.apps.shutil.which") def test_miniforge_version_check(self, mock_which, mock_run): - def which_side_effect(name): + def which_side_effect(name, **kwargs): if name == "miniforge": return None if name == "conda": @@ -116,7 +116,7 @@ def which_side_effect(name): @patch("fileglancer.apps.subprocess.run") @patch("fileglancer.apps.shutil.which") def test_miniforge_version_too_old(self, mock_which, mock_run): - def which_side_effect(name): + def which_side_effect(name, **kwargs): if name == "miniforge": return None if name == "conda": From 30325140c5993a98040a676d176a989fc6f1f7e7 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sun, 1 Mar 2026 07:35:07 -0500 Subject: [PATCH 18/20] fixed output refresh --- frontend/src/components/JobDetail.tsx | 5 +++-- .../src/components/ui/AppsPage/AppLaunchForm.tsx | 14 +++++++++++++- frontend/src/queries/jobsQueries.ts | 10 ++++++---- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/frontend/src/components/JobDetail.tsx b/frontend/src/components/JobDetail.tsx index c1706d36..429c3dd3 100644 --- a/frontend/src/components/JobDetail.tsx +++ b/frontend/src/components/JobDetail.tsx @@ -145,9 +145,10 @@ export default function JobDetail() { const id = jobId ? parseInt(jobId) : 0; const jobQuery = useJobQuery(id); + const jobStatus = jobQuery.data?.status; const scriptQuery = useJobFileQuery(id, 'script'); - const stdoutQuery = useJobFileQuery(id, 'stdout'); - const stderrQuery = useJobFileQuery(id, 'stderr'); + const stdoutQuery = useJobFileQuery(id, 'stdout', jobStatus); + const stderrQuery = useJobFileQuery(id, 'stderr', jobStatus); const cancelMutation = useCancelJobMutation(); const isService = jobQuery.data?.entry_point_type === 'service'; diff --git a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx index d3e07e03..f34c9a4c 100644 --- a/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx +++ b/frontend/src/components/ui/AppsPage/AppLaunchForm.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import type { Dispatch, SetStateAction } from 'react'; import { Accordion, Button, Tabs, Typography } from '@material-tailwind/react'; @@ -702,6 +702,18 @@ export default function AppLaunchForm({ ); const [extraArgs, setExtraArgs] = useState(resolvedExtraArgs); + // Update extraArgs when async data (preferences or cluster defaults) arrives, + // but only if not a relaunch and the user hasn't modified the field yet + useEffect(() => { + if (externalExtraArgs !== undefined) { + return; // relaunch value takes priority, don't overwrite + } + const resolved = defaultExtraArgs || configExtraArgs; + if (resolved) { + setExtraArgs(prev => (prev === '' ? resolved : prev)); + } + }, [defaultExtraArgs, configExtraArgs, externalExtraArgs]); + // Environment tab state — relaunch values override entry point defaults const [envVars, setEnvVars] = useState(() => { const source = initialEnv ?? entryPoint.env; diff --git a/frontend/src/queries/jobsQueries.ts b/frontend/src/queries/jobsQueries.ts index 87b8ed10..3c8a51ab 100644 --- a/frontend/src/queries/jobsQueries.ts +++ b/frontend/src/queries/jobsQueries.ts @@ -140,14 +140,16 @@ export function useJobQuery(jobId: number): UseQueryResult { export function useJobFileQuery( jobId: number, - fileType: string + fileType: string, + jobStatus?: string ): UseQueryResult { + const isActive = jobStatus === 'PENDING' || jobStatus === 'RUNNING'; return useQuery({ queryKey: [...jobsQueryKeys.detail(jobId), 'file', fileType], queryFn: ({ signal }) => fetchJobFile(jobId, fileType, signal), - refetchInterval: query => { - // Only auto-refresh if file doesn't exist yet (null) - it may appear later - return query.state.data === null ? 10000 : false; + refetchInterval: () => { + // Auto-refresh while job is active, or if file doesn't exist yet + return isActive ? 5000 : false; } }); } From fefcd9530b6be8476bcc19b3719d83db929d3249 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sun, 1 Mar 2026 10:08:44 -0500 Subject: [PATCH 19/20] add positional args last --- fileglancer/apps.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/fileglancer/apps.py b/fileglancer/apps.py index a594367e..0b4e3040 100644 --- a/fileglancer/apps.py +++ b/fileglancer/apps.py @@ -391,29 +391,29 @@ def build_command(entry_point: AppEntryPoint, parameters: dict) -> str: # Start with the base command parts = [entry_point.command] - # Pass 1: Positional args in declaration order + # Pass 1: Flagged args in declaration order for param in flat_params: - if param.flag is not None: + if param.flag is None: continue if param.key not in effective: continue p, value = effective[param.key] validated = _validate_parameter_value(p, value) - parts.append(shlex.quote(validated)) + if p.type == "boolean": + if value is True: + parts.append(p.flag) + else: + parts.append(f"{p.flag} {shlex.quote(validated)}") - # Pass 2: Flagged args in declaration order + # Pass 2: Positional args in declaration order for param in flat_params: - if param.flag is None: + if param.flag is not None: continue if param.key not in effective: continue p, value = effective[param.key] validated = _validate_parameter_value(p, value) - if p.type == "boolean": - if value is True: - parts.append(p.flag) - else: - parts.append(f"{p.flag} {shlex.quote(validated)}") + parts.append(shlex.quote(validated)) return (" \\\n ").join(parts) @@ -840,7 +840,7 @@ async def submit_job( db_job = db.get_job(session, job_id, username) session.expunge(db_job) - logger.info(f"Job {db_job.id} submitted for user {username} in {work_dir}: {command}") + logger.info(f"Job {db_job.id} submitted for user {username} in {work_dir}") return db_job From 42b5acfcc1ad2daf24d555182c11787c24d44e17 Mon Sep 17 00:00:00 2001 From: Konrad Rokicki Date: Sun, 1 Mar 2026 10:38:25 -0500 Subject: [PATCH 20/20] fix lint errors --- frontend/src/components/ui/PreferencesPage/JobOptions.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/ui/PreferencesPage/JobOptions.tsx b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx index ac2575d6..adc82b3f 100644 --- a/frontend/src/components/ui/PreferencesPage/JobOptions.tsx +++ b/frontend/src/components/ui/PreferencesPage/JobOptions.tsx @@ -84,7 +84,8 @@ export default function JobOptions() { Container cache directory - Directory where Apptainer SIF images are cached. Defaults to ~/.fileglancer/apptainer_cache if not set. + Directory where Apptainer SIF images are cached. Defaults to{' '} + ~/.fileglancer/apptainer_cache if not set.