Issue 564: add size filtering and typed min access level

Support excluding GitLab projects by statistics-backed size bounds and pass through minAccessLevel for project listing with AccessLevel-aligned typing.

Author: WilliamZhang9

packages/backend/src/gitlab.test.ts

@@ -68,3 +68,73 @@ test('shouldExcludeProject returns false when exclude.userOwnedProjects is true
         exclude: { userOwnedProjects: true },
     })).toBe(false);
 });
+
+test('shouldExcludeProject returns true when project size is less than exclude.size.min.', () => {
+    const project = {
+        path_with_namespace: 'test/project',
+        statistics: {
+            storage_size: 99,
+        },
+    } as unknown as ProjectSchema;
+
+    expect(shouldExcludeProject({
+        project,
+        exclude: {
+            size: {
+                min: 100,
+            },
+        },
+    })).toBe(true);
+});
+
+test('shouldExcludeProject returns true when project size is greater than exclude.size.max.', () => {
+    const project = {
+        path_with_namespace: 'test/project',
+        statistics: {
+            storage_size: 101,
+        },
+    } as unknown as ProjectSchema;
+
+    expect(shouldExcludeProject({
+        project,
+        exclude: {
+            size: {
+                max: 100,
+            },
+        },
+    })).toBe(true);
+});
+
+test('shouldExcludeProject returns false when project size is within exclude.size bounds.', () => {
+    const project = {
+        path_with_namespace: 'test/project',
+        statistics: {
+            storage_size: 100,
+        },
+    } as unknown as ProjectSchema;
+
+    expect(shouldExcludeProject({
+        project,
+        exclude: {
+            size: {
+                min: 100,
+                max: 100,
+            },
+        },
+    })).toBe(false);
+});
+
+test('shouldExcludeProject returns false when exclude.size is set but project statistics are unavailable.', () => {
+    const project = {
+        path_with_namespace: 'test/project',
+    } as ProjectSchema;
+
+    expect(shouldExcludeProject({
+        project,
+        exclude: {
+            size: {
+                min: 100,
+            },
+        },
+    })).toBe(false);
+});

packages/backend/src/gitlab.ts

@@ -11,6 +11,17 @@ import { fetchWithRetry, measure } from "./utils.js";
 const logger = createLogger('gitlab');
 export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
 
+export enum AccessLevel {
+    MINIMAL_ACCESS = 5,
+    GUEST = 10,
+    REPORTER = 20,
+    DEVELOPER = 30,
+    MAINTAINER = 40,
+    OWNER = 50,
+}
+
+type ProjectsAccessLevel = AccessLevel;
+
 export const createGitLabFromPersonalAccessToken = async ({ token, url }: { token?: string, url?: string }) => {
     const isGitLabCloud = url ? new URL(url).hostname === GITLAB_CLOUD_HOSTNAME : true;
     return new Gitlab({
@@ -48,6 +59,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
         token,
         url: config.url,
     });
+    const minAccessLevel: ProjectsAccessLevel | undefined = config.minAccessLevel;
+    const projectListOptions = {
+        perPage: 100,
+        ...(minAccessLevel !== undefined ? {
+            minAccessLevel,
+        } : {}),
+        ...(config.exclude?.size ? {
+            statistics: true,
+        } : {}),
+    };
 
     let allRepos: ProjectSchema[] = [];
     let allWarnings: string[] = [];
@@ -58,7 +79,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
                 logger.debug(`Fetching all projects visible in ${config.url}...`);
                 const { durationMs, data: _projects } = await measure(async () => {
                     const fetchFn = () => api.Projects.all({
-                        perPage: 100,
+                        ...projectListOptions,
                     });
                     return fetchWithRetry(fetchFn, `all projects in ${config.url}`, logger);
                 });
@@ -82,8 +103,8 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
                 logger.debug(`Fetching project info for group ${group}...`);
                 const { durationMs, data } = await measure(async () => {
                     const fetchFn = () => api.Groups.allProjects(group, {
-                        perPage: 100,
-                        includeSubgroups: true
+                        ...projectListOptions,
+                        includeSubgroups: true,
                     });
                     return fetchWithRetry(fetchFn, `group ${group}`, logger);
                 });
@@ -123,7 +144,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
                 logger.debug(`Fetching project info for user ${user}...`);
                 const { durationMs, data } = await measure(async () => {
                     const fetchFn = () => api.Users.allProjects(user, {
-                        perPage: 100,
+                        ...projectListOptions,
                     });
                     return fetchWithRetry(fetchFn, `user ${user}`, logger);
                 });
@@ -162,7 +183,9 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) =
             try {
                 logger.debug(`Fetching project info for project ${project}...`);
                 const { durationMs, data } = await measure(async () => {
-                    const fetchFn = () => api.Projects.show(project);
+                    const fetchFn = () => api.Projects.show(project, {
+                        statistics: config.exclude?.size ? true : undefined,
+                    });
                     return fetchWithRetry(fetchFn, `project ${project}`, logger);
                 });
                 logger.debug(`Found project ${project} in ${durationMs}ms.`);
@@ -253,6 +276,21 @@ export const shouldExcludeProject = ({
             }
         }
 
+        if (exclude?.size) {
+            const projectSizeBytes = getProjectSizeBytes(project);
+            if (projectSizeBytes !== undefined) {
+                if (exclude.size.min !== undefined && projectSizeBytes < exclude.size.min) {
+                    reason = `project size (${projectSizeBytes}) is less than \`exclude.size.min\` (${exclude.size.min})`;
+                    return true;
+                }
+
+                if (exclude.size.max !== undefined && projectSizeBytes > exclude.size.max) {
+                    reason = `project size (${projectSizeBytes}) is greater than \`exclude.size.max\` (${exclude.size.max})`;
+                    return true;
+                }
+            }
+        }
+
         if (include?.topics) {
             const configTopics = include.topics.map(topic => topic.toLowerCase());
             const projectTopics = project.topics ?? [];
@@ -284,6 +322,39 @@ export const shouldExcludeProject = ({
     return false;
 }
 
+const getProjectSizeBytes = (project: ProjectSchema): number | undefined => {
+    // GitLab's API returns size data in the statistics object when `statistics=true`.
+    // We support both snake_case and camelCase keys to be resilient to response typing differences.
+    const projectWithStats = project as ProjectSchema & {
+        statistics?: {
+            storage_size?: number;
+            repository_size?: number;
+            storageSize?: number;
+            repositorySize?: number;
+        };
+    };
+
+    const statistics = projectWithStats.statistics;
+    if (!statistics) {
+        return;
+    }
+
+    if (typeof statistics.storage_size === "number") {
+        return statistics.storage_size;
+    }
+    if (typeof statistics.repository_size === "number") {
+        return statistics.repository_size;
+    }
+    if (typeof statistics.storageSize === "number") {
+        return statistics.storageSize;
+    }
+    if (typeof statistics.repositorySize === "number") {
+        return statistics.repositorySize;
+    }
+
+    return;
+}
+
 export const getProjectMembers = async (projectId: string, api: InstanceType<typeof Gitlab>) => {
     try {
         const fetchFn = () => api.ProjectMembers.all(projectId, {

packages/schemas/src/v3/connection.schema.ts

@@ -290,6 +290,18 @@ const schema = {
           ],
           "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
         },
+        "minAccessLevel": {
+          "type": "integer",
+          "enum": [
+            5,
+            10,
+            20,
+            30,
+            40,
+            50
+          ],
+          "description": "Minimum GitLab access level required for projects to be returned. Uses GitLab role levels where 20=Reporter, 30=Developer, 40=Maintainer, and 50=Owner."
+        },
         "projects": {
           "type": "array",
           "items": {
@@ -362,6 +374,21 @@ const schema = {
                   "ci"
                 ]
               ]
+            },
+            "size": {
+              "type": "object",
+              "description": "Exclude projects based on GitLab statistics size fields (in bytes).",
+              "properties": {
+                "min": {
+                  "type": "integer",
+                  "description": "Minimum project size (in bytes) to sync (inclusive). Projects smaller than this will be excluded."
+                },
+                "max": {
+                  "type": "integer",
+                  "description": "Maximum project size (in bytes) to sync (inclusive). Projects larger than this will be excluded."
+                }
+              },
+              "additionalProperties": false
             }
           },
           "additionalProperties": false

packages/schemas/src/v3/connection.type.ts

@@ -135,6 +135,10 @@ export interface GitlabConnectionConfig {
    * List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`).
    */
   groups?: string[];
+  /**
+   * Minimum GitLab access level required for projects to be returned. Uses GitLab role levels where 20=Reporter, 30=Developer, 40=Maintainer, and 50=Owner.
+   */
+  minAccessLevel?: 5 | 10 | 20 | 30 | 40 | 50;
   /**
    * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/
    */
@@ -166,6 +170,19 @@ export interface GitlabConnectionConfig {
      * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
      */
     topics?: string[];
+    /**
+     * Exclude projects based on GitLab statistics size fields (in bytes).
+     */
+    size?: {
+      /**
+       * Minimum project size (in bytes) to sync (inclusive). Projects smaller than this will be excluded.
+       */
+      min?: number;
+      /**
+       * Maximum project size (in bytes) to sync (inclusive). Projects larger than this will be excluded.
+       */
+      max?: number;
+    };
   };
   revisions?: GitRevisions;
 }

packages/schemas/src/v3/gitlab.schema.ts

@@ -78,6 +78,18 @@ const schema = {
       ],
       "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
     },
+    "minAccessLevel": {
+      "type": "integer",
+      "enum": [
+        5,
+        10,
+        20,
+        30,
+        40,
+        50
+      ],
+      "description": "Minimum GitLab access level required for projects to be returned. Uses GitLab role levels where 20=Reporter, 30=Developer, 40=Maintainer, and 50=Owner."
+    },
     "projects": {
       "type": "array",
       "items": {
@@ -150,6 +162,21 @@ const schema = {
               "ci"
             ]
           ]
+        },
+        "size": {
+          "type": "object",
+          "description": "Exclude projects based on GitLab statistics size fields (in bytes).",
+          "properties": {
+            "min": {
+              "type": "integer",
+              "description": "Minimum project size (in bytes) to sync (inclusive). Projects smaller than this will be excluded."
+            },
+            "max": {
+              "type": "integer",
+              "description": "Maximum project size (in bytes) to sync (inclusive). Projects larger than this will be excluded."
+            }
+          },
+          "additionalProperties": false
         }
       },
       "additionalProperties": false

packages/schemas/src/v3/gitlab.type.ts

@@ -37,6 +37,10 @@ export interface GitlabConnectionConfig {
    * List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`).
    */
   groups?: string[];
+  /**
+   * Minimum GitLab access level required for projects to be returned. Uses GitLab role levels where 20=Reporter, 30=Developer, 40=Maintainer, and 50=Owner.
+   */
+  minAccessLevel?: 5 | 10 | 20 | 30 | 40 | 50;
   /**
    * List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/
    */
@@ -68,6 +72,19 @@ export interface GitlabConnectionConfig {
      * List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.
      */
     topics?: string[];
+    /**
+     * Exclude projects based on GitLab statistics size fields (in bytes).
+     */
+    size?: {
+      /**
+       * Minimum project size (in bytes) to sync (inclusive). Projects smaller than this will be excluded.
+       */
+      min?: number;
+      /**
+       * Maximum project size (in bytes) to sync (inclusive). Projects larger than this will be excluded.
+       */
+      max?: number;
+    };
   };
   revisions?: GitRevisions;
 }

packages/schemas/src/v3/index.schema.ts

@@ -705,6 +705,18 @@ const schema = {
                   ],
                   "description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
                 },
+                "minAccessLevel": {
+                  "type": "integer",
+                  "enum": [
+                    5,
+                    10,
+                    20,
+                    30,
+                    40,
+                    50
+                  ],
+                  "description": "Minimum GitLab access level required for projects to be returned. Uses GitLab role levels where 20=Reporter, 30=Developer, 40=Maintainer, and 50=Owner."
+                },
                 "projects": {
                   "type": "array",
                   "items": {
@@ -777,6 +789,21 @@ const schema = {
                           "ci"
                         ]
                       ]
+                    },
+                    "size": {
+                      "type": "object",
+                      "description": "Exclude projects based on GitLab statistics size fields (in bytes).",
+                      "properties": {
+                        "min": {
+                          "type": "integer",
+                          "description": "Minimum project size (in bytes) to sync (inclusive). Projects smaller than this will be excluded."
+                        },
+                        "max": {
+                          "type": "integer",
+                          "description": "Maximum project size (in bytes) to sync (inclusive). Projects larger than this will be excluded."
+                        }
+                      },
+                      "additionalProperties": false
                     }
                   },
                   "additionalProperties": false