feat(submit): add KCIDB build submission subcommand and payload helpers

docs/config_file.md

@@ -29,16 +29,20 @@ token="example"
 pipeline="https://staging.kernelci.org:9100/"
 api="https://staging.kernelci.org:9000/"
 token="example"
+kcidb_rest_url="https://db.kernelci.org/submit"
+kcidb_token="your-kcidb-token-here"
 
 [production]
 pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
 api="https://kernelci-api.westus3.cloudapp.azure.com/"
 token="example"
+kcidb_rest_url="https://staging.kcidb.kernelci.org/submit"
+kcidb_token="your-kcidb-token-here"
 ```
 
 Where `default_instance` is the default instance to use, if not provided in the command line.  
 In section `local`, `staging`, `production` you can provide the host for the pipeline, api and also a token for the available instances.  
-pipeline is the URL of the KernelCI Pipeline API endpoint, api is the URL of the new KernelCI API endpoint, and token is the API token to use for authentication.  
-If you are using KernelCI Pipeline instance, you can get the token from the project maintainers.  
+`pipeline` is the URL of the KernelCI Pipeline API endpoint, `api` is the URL of the new KernelCI API endpoint, and `token` is the API token to use for authentication. `kcidb_rest_uri` is KCIDB submission endpoint, and `kcidb_token` is the API token to use for authentication with KCIDB.   
+If you are using KernelCI instances of pipeline or/and KCIDB, you can get the token from the KernelCI project maintainers.   
 If it is a local instance, you can generate your token using [kernelci-pipeline/tools/jwt_generator.py](https://github.com/kernelci/kernelci-pipeline/blob/main/tools/jwt_generator.py) script.  
 

kcidev/_data/kci-dev.toml.example

@@ -9,8 +9,12 @@ token="example"
 pipeline="https://staging.kernelci.org:9100/"
 api="https://staging.kernelci.org:9000/"
 token="example"
+kcidb_rest_url="https://staging.kcidb.kernelci.org/submit"
+kcidb_token="your-kcidb-token-here"
 
 [production]
 pipeline="https://kernelci-pipeline.westus3.cloudapp.azure.com/"
 api="https://kernelci-api.westus3.cloudapp.azure.com/"
-token="example"
+token="example"
+kcidb_rest_url="https://db.kernelci.org/submit"
+kcidb_token="your-kcidb-token-here"

kcidev/libs/common.py

@@ -78,7 +78,7 @@ def load_toml(settings, subcommand):
         return config
 
     # config and results subcommand work without a config file
-    if subcommand != "config" and subcommand != "results":
+    if subcommand not in ("config", "results", "submit"):
         if not config:
             logging.warning(f"No config file found for subcommand {subcommand}")
             kci_err(

kcidev/libs/kcidb.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import hashlib
+import json
+import os
+import urllib.parse
+
+import click
+
+from kcidev.libs.common import kci_err, kci_msg, kci_warning, kcidev_session
+
+KCIDB_SCHEMA_MAJOR = 5
+KCIDB_SCHEMA_MINOR = 3
+
+
+# ---------------------
+# ID generation
+# ---------------------
+
+
+def _sha256_hex(data):
+    return hashlib.sha256(data.encode("utf-8")).hexdigest()
+
+
+def generate_checkout_id(origin, repo_url, branch, commit, patchset_hash=""):
+    """
+    Deterministic checkout ID from source identity.
+    Format: origin:ck-<sha256(repo_url|branch|commit|patchset_hash)>
+    """
+    seed = f"{repo_url}|{branch}|{commit}|{patchset_hash}"
+    return f"{origin}:ck-{_sha256_hex(seed)}"
+
+
+def generate_build_id(origin, checkout_id, arch, config_name, compiler, start_time):
+    """
+    Deterministic build ID from build parameters.
+    Format: origin:b-<sha256(checkout_id|arch|config_name|compiler|start_time)>
+    """
+    seed = f"{checkout_id}|{arch}|{config_name}|{compiler}|{start_time}"
+    return f"{origin}:b-{_sha256_hex(seed)}"
+
+
+# ---------------------
+# Payload building
+# ---------------------
+
+
+def build_checkout_payload(origin, checkout_id, **kwargs):
+    """Build a checkout object for the KCIDB JSON payload."""
+    checkout = {
+        "id": checkout_id,
+        "origin": origin,
+    }
+    for key in [
+        "tree_name",
+        "git_repository_url",
+        "git_repository_branch",
+        "git_commit_hash",
+        "patchset_hash",
+        "start_time",
+        "valid",
+    ]:
+        if kwargs.get(key) is not None:
+            checkout[key] = kwargs[key]
+    return checkout
+
+
+def build_build_payload(origin, build_id, checkout_id, **kwargs):
+    """Build a build object for the KCIDB JSON payload."""
+    build = {
+        "id": build_id,
+        "origin": origin,
+        "checkout_id": checkout_id,
+    }
+    for key in [
+        "start_time",
+        "duration",
+        "architecture",
+        "compiler",
+        "config_name",
+        "config_url",
+        "log_url",
+        "comment",
+        "command",
+        "status",
+    ]:
+        if kwargs.get(key) is not None:
+            build[key] = kwargs[key]
+    return build
+
+
+def build_submission_payload(checkouts, builds):
+    """Build the complete KCIDB v5.3 submission JSON."""
+    payload = {
+        "version": {"major": KCIDB_SCHEMA_MAJOR, "minor": KCIDB_SCHEMA_MINOR},
+    }
+    if checkouts:
+        payload["checkouts"] = checkouts
+    if builds:
+        payload["builds"] = builds
+    return payload
+
+
+# ---------------------
+# Config resolution
+# ---------------------
+
+
+def _parse_kcidb_rest_env(env_value):
+    """
+    Parse KCIDB_REST env var.
+    Format: https://token@host[:port][/path]
+    Returns (url, token) or (None, None) on failure.
+    """
+    parsed = urllib.parse.urlparse(env_value)
+    token = parsed.username
+    if not token:
+        return None, None
+
+    # Rebuild URL without credentials
+    host = parsed.hostname
+    if parsed.port:
+        host = f"{host}:{parsed.port}"
+    path = parsed.path if parsed.path else "/"
+    if not path.endswith("/submit"):
+        path = path.rstrip("/") + "/submit"
+
+    clean_url = urllib.parse.urlunparse(
+        (parsed.scheme, host, path, parsed.params, parsed.query, parsed.fragment)
+    )
+    return clean_url, token
+
+
+def resolve_kcidb_config(cfg, instance, cli_rest_url, cli_token):
+    """
+    Resolve KCIDB REST URL and token. Priority:
+    1. CLI flags (--kcidb-rest-url, --kcidb-token)
+    2. KCIDB_REST environment variable
+    3. Instance config (kcidb_rest_url, kcidb_token in TOML)
+
+    Returns (rest_url, token) tuple.
+    Raises click.Abort if no valid credentials found.
+    """
+    # 1. CLI flags
+    if cli_rest_url or cli_token:
+        if not cli_rest_url or not cli_token:
+            kci_err("Both --kcidb-rest-url and --kcidb-token must be provided together")
+            raise click.Abort()
+        return cli_rest_url, cli_token
+
+    # 2. Environment variable
+    kcidb_rest_env = os.environ.get("KCIDB_REST")
+    if kcidb_rest_env:
+        url, token = _parse_kcidb_rest_env(kcidb_rest_env)
+        if url and token:
+            return url, token
+        kci_err("KCIDB_REST env var set but could not parse token from it")
+
+    # 3. Instance config
+    if cfg and instance and instance in cfg:
+        inst_cfg = cfg[instance]
+        rest_url = inst_cfg.get("kcidb_rest_url")
+        token = inst_cfg.get("kcidb_token")
+        if rest_url and token:
+            return rest_url, token
+
+    kci_err(
+        "No KCIDB credentials found. Provide --kcidb-rest-url and --kcidb-token, "
+        "set KCIDB_REST env var, or configure kcidb_rest_url/kcidb_token in config file"
+    )
+    raise click.Abort()
+
+
+# ---------------------
+# REST submission
+# ---------------------
+
+
+def submit_to_kcidb(rest_url, token, payload, timeout=60):
+    """
+    POST the JSON payload to the KCIDB REST API.
+
+    Returns the response text on success.
+    Raises click.Abort on failure.
+    """
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {token}",
+    }
+    try:
+        response = kcidev_session.post(
+            rest_url, json=payload, headers=headers, timeout=timeout
+        )
+    except Exception as e:
+        kci_err(f"KCIDB API connection error: {e}")
+        raise click.Abort()
+
+    if response.status_code < 300:
+        try:
+            return response.json()
+        except Exception:
+            return response.text
+    else:
+        kci_err(f"KCIDB submission failed: HTTP {response.status_code}")
+        try:
+            kci_err(response.json())
+        except Exception:
+            kci_err(response.text)
+        raise click.Abort()

kcidev/main.py

@@ -13,6 +13,7 @@
     config,
     maestro,
     results,
+    submit,
     testretry,
     watch,
 )
@@ -40,19 +41,18 @@ def cli(ctx, settings, instance, debug):
     subcommand = ctx.invoked_subcommand
     ctx.obj = {"CFG": load_toml(settings, subcommand)}
     ctx.obj["SETTINGS"] = settings
-    if subcommand != "results" and subcommand != "config":
+    if subcommand not in ("results", "config"):
         if instance:
             ctx.obj["INSTANCE"] = instance
-        else:
+        elif subcommand != "submit":
             ctx.obj["INSTANCE"] = ctx.obj["CFG"].get("default_instance")
-        fconfig = config_path(settings)
-        if not ctx.obj["INSTANCE"]:
-            kci_err(f"No instance defined in settings or as argument in {fconfig}")
-            raise click.Abort()
-        if ctx.obj["INSTANCE"] not in ctx.obj["CFG"]:
-            kci_err(f"Instance {ctx.obj['INSTANCE']} not found in {fconfig}")
-            raise click.Abort()
-        pass
+            fconfig = config_path(settings)
+            if not ctx.obj["INSTANCE"]:
+                kci_err(f"No instance defined in settings or as argument in {fconfig}")
+                raise click.Abort()
+            if ctx.obj["INSTANCE"] not in ctx.obj["CFG"]:
+                kci_err(f"Instance {ctx.obj['INSTANCE']} not found in {fconfig}")
+                raise click.Abort()
 
 
 def run():
@@ -63,6 +63,7 @@ def run():
     cli.add_command(maestro.maestro)
     cli.add_command(testretry.testretry)
     cli.add_command(results.results)
+    cli.add_command(submit.submit)
     cli.add_command(watch.watch)
     cli()