Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ _mytests
.idea
.zed
docs/changelog.md
.generated

# Yarn files
.yarn/install-state.gz
Expand Down
22 changes: 22 additions & 0 deletions docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,8 @@ SUBCOMMANDS
actor calculate-memory Calculates the Actor’s dynamic
memory usage based on a memory expression from
actor.json, input data, and run options.
actor generate-types Generate TypeScript types from a
JSON schema file.
```

##### `apify actor calculate-memory`
Expand Down Expand Up @@ -491,6 +493,26 @@ FLAGS
charging without actually charging
```

##### `apify actor generate-types`

```sh
DESCRIPTION
Generate TypeScript types from a JSON schema file.

USAGE
$ apify actor generate-types <path> [-o <value>]
[-s]

ARGUMENTS
path Path to the JSON schema file.

FLAGS
-o, --output=<value> Directory where the generated files
should be outputted.
-s, --strict Whether generated interfaces should be
strict (no index signature [key: string]: unknown).
```

##### `apify actor get-input`

```sh
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
"istextorbinary": "~9.5.0",
"jju": "~1.4.0",
"js-levenshtein": "^1.1.6",
"json-schema-to-typescript": "^15.0.4",
"lodash.clonedeep": "^4.5.0",
"mime": "~4.1.0",
"open": "~11.0.0",
Expand Down
1 change: 1 addition & 0 deletions scripts/generate-cli-docs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ const categories: Record<string, CommandsInCategory[]> = {
{ command: Commands.actor },
{ command: Commands.actorCalculateMemory },
{ command: Commands.actorCharge },
{ command: Commands.actorGenerateTypes },
{ command: Commands.actorGetInput },
{ command: Commands.actorGetPublicUrl },
{ command: Commands.actorGetValue },
Expand Down
2 changes: 2 additions & 0 deletions src/commands/_register.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { BuiltApifyCommand } from '../lib/command-framework/apify-command.j
import { ActorIndexCommand } from './actor/_index.js';
import { ActorCalculateMemoryCommand } from './actor/calculate-memory.js';
import { ActorChargeCommand } from './actor/charge.js';
import { ActorGenerateTypesCommand } from './actor/generate-types.js';
import { ActorGetInputCommand } from './actor/get-input.js';
import { ActorGetPublicUrlCommand } from './actor/get-public-url.js';
import { ActorGetValueCommand } from './actor/get-value.js';
Expand Down Expand Up @@ -77,6 +78,7 @@ export const actorCommands = [
ActorGetInputCommand,
ActorChargeCommand,
ActorCalculateMemoryCommand,
ActorGenerateTypesCommand,

// top-level
HelpCommand,
Expand Down
2 changes: 2 additions & 0 deletions src/commands/actor/_index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
import { ActorCalculateMemoryCommand } from './calculate-memory.js';
import { ActorChargeCommand } from './charge.js';
import { ActorGenerateTypesCommand } from './generate-types.js';
import { ActorGetInputCommand } from './get-input.js';
import { ActorGetPublicUrlCommand } from './get-public-url.js';
import { ActorGetValueCommand } from './get-value.js';
Expand All @@ -21,6 +22,7 @@ export class ActorIndexCommand extends ApifyCommand<typeof ActorIndexCommand> {
ActorGetInputCommand,
ActorChargeCommand,
ActorCalculateMemoryCommand,
ActorGenerateTypesCommand,
];

async run() {
Expand Down
89 changes: 89 additions & 0 deletions src/commands/actor/generate-types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';

import type { JSONSchema4 } from 'json-schema';
import { compile } from 'json-schema-to-typescript';

import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
import { Args } from '../../lib/command-framework/args.js';
import { Flags } from '../../lib/command-framework/flags.js';
import { LOCAL_CONFIG_PATH } from '../../lib/consts.js';
import { readAndValidateInputSchema } from '../../lib/input_schema.js';
import { success } from '../../lib/outputs.js';

export const BANNER_COMMENT = `
/* eslint-disable */
/* biome-ignore-all lint */
/* biome-ignore-all format */
/* prettier-ignore-start */
/**
* This file was automatically generated by json-schema-to-typescript.
* DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
* and run apify actor generate-types to regenerate this file.
*/
/** */
`;

export class ActorGenerateTypesCommand extends ApifyCommand<typeof ActorGenerateTypesCommand> {
static override name = 'generate-types' as const;

static override description = `Generate TypeScript types from an Actor input schema.

Reads the input schema from one of these locations (in priority order):
1. Object in '${LOCAL_CONFIG_PATH}' under "input" key
2. JSON file path in '${LOCAL_CONFIG_PATH}' "input" key
3. .actor/INPUT_SCHEMA.json
4. INPUT_SCHEMA.json

Optionally specify custom schema path to use.`;

static override flags = {
output: Flags.string({
char: 'o',
description: 'Directory where the generated files should be outputted.',
required: false,
default: 'src/.generated/actor/',
}),
strict: Flags.boolean({
char: 's',
description: 'Whether generated interfaces should be strict (no index signature [key: string]: unknown).',
required: false,
default: true,
}),
};

static override args = {
path: Args.string({
required: false,
description: 'Optional path to the input schema file. If not provided, searches default locations.',
}),
};

async run() {
const { inputSchema, inputSchemaPath } = await readAndValidateInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
action: 'Generating types from',
});
Comment on lines +64 to +68
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

generate-type share a validation logic with validate-schema now


const name = inputSchemaPath ? path.basename(inputSchemaPath, path.extname(inputSchemaPath)) : 'input';

const result = await compile(inputSchema as JSONSchema4, name, {
bannerComment: BANNER_COMMENT,
maxItems: -1,
unknownAny: true,
format: true,
additionalProperties: !this.flags.strict,
$refOptions: { resolve: { external: false, file: false, http: false } },
});

const outputDir = path.resolve(process.cwd(), this.flags.output);
await mkdir(outputDir, { recursive: true });

const outputFile = path.join(outputDir, `${name}.ts`);
await writeFile(outputFile, result, 'utf-8');

success({ message: `Generated types written to ${outputFile}` });
}
}
22 changes: 4 additions & 18 deletions src/commands/validate-schema.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import process from 'node:process';

import { validateInputSchema } from '@apify/input_schema';

import { ApifyCommand } from '../lib/command-framework/apify-command.js';
import { Args } from '../lib/command-framework/args.js';
import { LOCAL_CONFIG_PATH } from '../lib/consts.js';
import { readInputSchema } from '../lib/input_schema.js';
import { info, success } from '../lib/outputs.js';
import { Ajv2019 } from '../lib/utils.js';
import { readAndValidateInputSchema } from '../lib/input_schema.js';
import { success } from '../lib/outputs.js';

export class ValidateInputSchemaCommand extends ApifyCommand<typeof ValidateInputSchemaCommand> {
static override name = 'validate-schema' as const;
Expand All @@ -30,23 +27,12 @@ Optionally specify custom schema path to validate.`;
static override hiddenAliases = ['vis'];

async run() {
const { inputSchema, inputSchemaPath } = await readInputSchema({
await readAndValidateInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
action: 'Validating',
});

if (!inputSchema) {
throw new Error(`Input schema has not been found at ${inputSchemaPath}.`);
}

if (inputSchemaPath) {
info({ message: `Validating input schema stored at ${inputSchemaPath}` });
} else {
info({ message: `Validating input schema embedded in '${LOCAL_CONFIG_PATH}'` });
}

const validator = new Ajv2019({ strict: false });
validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema.
success({ message: 'Input schema is valid.' });
}
}
43 changes: 41 additions & 2 deletions src/lib/input_schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import deepClone from 'lodash.clonedeep';
import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
import { validateInputSchema } from '@apify/input_schema';

import { ACTOR_SPECIFICATION_FOLDER } from './consts.js';
import { warning } from './outputs.js';
import { ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_PATH } from './consts.js';
import { info, warning } from './outputs.js';
import { Ajv2019, getJsonFileContent, getLocalConfig, getLocalKeyValueStorePath } from './utils.js';

const DEFAULT_INPUT_SCHEMA_PATHS = [
Expand Down Expand Up @@ -70,6 +70,45 @@ export const readInputSchema = async (
};
};

/**
* Reads and validates input schema, logging appropriate info messages.
* Throws an error if the schema is not found or invalid.
*
* @param options.forcePath - Optional path to force reading from
* @param options.cwd - Current working directory
* @param options.action - Action description for the info message (e.g., "Validating", "Generating types from")
* @returns The validated input schema and its path
*/
export const readAndValidateInputSchema = async ({
forcePath,
cwd,
action,
}: {
forcePath?: string;
cwd: string;
action: string;
}): Promise<{ inputSchema: Record<string, unknown>; inputSchemaPath: string | null }> => {
const { inputSchema, inputSchemaPath } = await readInputSchema({
forcePath,
cwd,
});

if (!inputSchema) {
throw new Error(`Input schema has not been found at ${inputSchemaPath}.`);
}

if (inputSchemaPath) {
info({ message: `${action} input schema at ${inputSchemaPath}` });
} else {
info({ message: `${action} input schema embedded in '${LOCAL_CONFIG_PATH}'` });
}

const validator = new Ajv2019({ strict: false });
validateInputSchema(validator, inputSchema);

return { inputSchema, inputSchemaPath };
};

/**
* Goes to the Actor directory and creates INPUT.json file from the input schema prefills.

Expand Down
80 changes: 80 additions & 0 deletions test/__setup__/input-schemas/complex.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
{
"$schema": "https://apify-projects.github.io/actor-json-schemas/input.ide.json?v=0.6",
"title": "Example Actor Input Schema",
"description": "A sample input schema demonstrating different input field types.",
"type": "object",
"schemaVersion": 1,
"properties": {
"startUrls": {
"title": "Start URLs",
"description": "List of URLs to start crawling",
"type": "array",
"editor": "requestListSources",
"prefill": [
{ "url": "https://example.com" },
{ "url": "https://example.org" }
]
},
"searchQuery": {
"title": "Search query",
"description": "The keyword or phrase to search for",
"type": "string",
"editor": "textfield",
"minLength": 3,
"default": "apify"
},
"maxItems": {
"title": "Maximum items to fetch",
"description": "Limit the number of items the Actor will process",
"type": "integer",
"editor": "number",
"minimum": 1,
"default": 100
},
"includeImages": {
"title": "Include images",
"description": "Whether to include image data in the results",
"type": "boolean",
"editor": "checkbox",
"default": false
},
"crawlerType": {
"title": "Crawler type",
"description": "Select the crawling engine to use",
"type": "string",
"editor": "select",
"enum": ["cheerio", "puppeteer", "playwright"],
"enumTitles": [
"Cheerio crawler",
"Puppeteer browser",
"Playwright browser"
],
"default": "cheerio"
},
"proxyConfig": {
"title": "Proxy configuration",
"description": "Optional proxy settings to use while crawling",
"type": "object",
"editor": "json",
"properties": {
"useApifyProxy": {
"title": "Use Apify Proxy",
"description": "Enable Apify Proxy",
"type": "boolean"
},
"customProxyUrls": {
"title": "Custom proxy URLs",
"description": "List of custom proxy URLs",
"type": "array",
"editor": "json",
"items": {
"type": "string"
}
}
},
"required": ["useApifyProxy"],
"additionalProperties": false
}
},
"required": ["startUrls", "searchQuery"]
}
2 changes: 2 additions & 0 deletions test/__setup__/input-schemas/paths.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ export const prefillsInputSchemaPath = fileURLToPath(new URL('./prefills.json',
export const unparsableInputSchemaPath = fileURLToPath(new URL('./unparsable.json', import.meta.url));

export const validInputSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url));

export const complexInputSchemaPath = fileURLToPath(new URL('./complex.json', import.meta.url));
Loading