diff --git a/README.md b/README.md index cfb2053..e368cad 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,8 @@ It's copy of our example file `config.json.sample`. More or less it looks like: | acl | - | String | Permission of S3 object. [See AWS ACL documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property). | | backup | - | Object | Backup original file setting. | | | bucket | String | Destination bucket to override. If not supplied, it will use `bucket` setting. | -| | directory | String | Image directory path. When starts with `./` relative to the source, otherwise creates a new tree. | +| | directory | String | Image directory path. Supports relative and absolute paths. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#directory) | +| | template | Object | Map representing pattern substitution pair. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#template) | | | prefix | String | Prepend filename prefix if supplied. | | | suffix | String | Append filename suffix if supplied. | | | acl | String | Permission of S3 object. [See AWS ACL documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property). | @@ -92,7 +93,8 @@ It's copy of our example file `config.json.sample`. More or less it looks like: | | quality | Number | Determine reduced image quality ( only `JPG` ). | | | jpegOptimizer | String | Determine optimiser that should be used `mozjpeg` (default) or `jpegoptim` ( only `JPG` ). | | | bucket | String | Destination bucket to override. If not supplied, it will use `bucket` setting. | -| | directory | String | Image directory path. When starts with `./` relative to the source, otherwise creates a new tree. | +| | directory | String | Image directory path. Supports relative and absolute paths. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#directory) | +| | template | Object | Map representing pattern substitution pair. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#template) | | | prefix | String | Prepend filename prefix if supplied. | | | suffix | String | Append filename suffix if supplied. | | | acl | String | Permission of S3 object. [See AWS ACL documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property). | @@ -105,7 +107,8 @@ It's copy of our example file `config.json.sample`. More or less it looks like: | | jpegOptimizer | String | Determine optimiser that should be used `mozjpeg` (default) or `jpegoptim` ( only `JPG` ). | | | orientation | Boolean | Auto orientation if value is `true`. | | | bucket | String | Destination bucket to override. If not supplied, it will use `bucket` setting. | -| | directory | String | Image directory path. When starts with `./` relative to the source, otherwise creates a new tree. | +| | directory | String | Image directory path. Supports relative and absolute paths. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#directory) | +| | template | Object | Map representing pattern substitution pair. Mode details in [DIRECTORY.md](doc/DIRECTORY.md/#template) | | | prefix | String | Prepend filename prefix if supplied. | | | suffix | String | Append filename suffix if supplied. | | | acl | String | Permission of S3 object. [See AWS ACL documentation](http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property). | diff --git a/bin/configtest b/bin/configtest index 4944e57..d14f645 100755 --- a/bin/configtest +++ b/bin/configtest @@ -56,7 +56,7 @@ var reset = '\u001b[0m'; stdout.write("--------------------------------\r\n"); if ( "backup" in config ) { var backup = config.backup || {}; - validateDestination(stdout, bucket, backup.bucket, backup.directory); + validateDestination(stdout, bucket, backup.bucket, backup.directory, backup.template); validatePrefixAndSuffix(stdout, backup.prefix, backup.suffix); validateAcl(stdout, acl, backup.acl); } else { @@ -70,7 +70,7 @@ var reset = '\u001b[0m'; var reduce = config.reduce || {}; validateQuality(stdout, reduce.quality); validateOptimizer(stdout, reduce.jpegOptimizer || jpegOptimizer); - validateDestination(stdout, bucket, reduce.bucket, reduce.directory); + validateDestination(stdout, bucket, reduce.bucket, reduce.directory, reduce.template); validatePrefixAndSuffix(stdout, reduce.prefix, reduce.suffix); validateAcl(stdout, acl, reduce.acl); } else { @@ -90,7 +90,7 @@ var reset = '\u001b[0m'; validateFormat(stdout, resize.format); validateQuality(stdout, resize.quality); validateOptimizer(stdout, resize.jpegOptimizer || jpegOptimizer); - validateDestination(stdout, bucket, resize.bucket, resize.directory); + validateDestination(stdout, bucket, resize.bucket, resize.directory, resize.template); validatePrefixAndSuffix(stdout, resize.prefix, resize.suffix); validateAcl(stdout, acl, resize.acl); stdout.write("\r\n"); @@ -158,9 +158,9 @@ var reset = '\u001b[0m'; } } - function validateDestination(stdout, globalBucket, bucket, directory) { + function validateDestination(stdout, globalBucket, bucket, directory, template) { var color = reset; - if ( ! bucket && ! globalBucket && (! directory || /^\.\//.test(directory))) { + if ( ! bucket && ! globalBucket && (! directory || /^\.\//.test(directory)) && (! template || ! template.pattern)) { warning.push(" Saving image to the same or relative directory may cause infinite Lambda process loop."); color = red; } @@ -172,6 +172,9 @@ var reset = '\u001b[0m'; if ( directory ) { stdout.write(directory); stdout.write( /^\.\.?/.test(directory) ? " [Relative]" : ""); + } else if ( template && template.pattern ) { + stdout.write(template.output || "/"); + stdout.write(" [Pattern]"); } else { stdout.write("[Same directory]"); } diff --git a/doc/DIRECTORY.md b/doc/DIRECTORY.md new file mode 100644 index 0000000..ba5c732 --- /dev/null +++ b/doc/DIRECTORY.md @@ -0,0 +1,143 @@ +# Directory configuration + +There are few ways of setting the output directory for processed files. All +of them work in the same way for resized, reduced and archived images. + +## Nothing + +You are allowed to choose to do not setup any output directory configuration and +use only `prefix` and/or `suffix` parameters. Just bare in mind that in such +case all output files will be saved in same directory as input file - +[S3 event notification limitations](#s3-event-notification-limitations). + +## Directory + +| Parameter | Type | Required | +|:---------:|:------:|:--------:| +| directory | String | no | + +`directory` parameter should be a `String` representing output path. It could be +an absolute (ie. `output/`) or relative (ie. `../output/`, `./output`) path. If +you decide to use relative path, bare in mind that this could lead to situation +where all output files will be saved in same directory structure as input file - +[S3 event notification limitations](#s3-event-notification-limitations). + +## Template + +| Parameter | Type | Required | +|:---------:|:------:|:--------:| +| template | Object | no | + +`template` parameter is a `Map` with two keys: `pattern` and `output`, ie.: + +``` +{ + template: { + pattern: "*path/c", + output: "*path/d" + } +} +``` + +`pattern` defines a pattern that describe path of input file directory. It's +used for matching and and parsing, which allows you to store parts of parsed +input directory as variables. More details in [Syntax](#template-syntax) +section. + +In case the input file directory will not match the `pattern`, it will be +skipped and the [`directory`](#directory) parameter will be processed, if +present. + +`output` defines a pattern that describe output directory path. It allows you to +reuse variables parsed from input directory, like in example above. More details +in [Syntax](#template-syntax) section. + +If you decide to use `template` parameter, bare in mind to avoid situation +where output files will be saved in same directory structure as input file - +[S3 event notification limitations](#s3-event-notification-limitations). + +### Template syntax + +**Source**: [path-template](https://github.com/matsadler/path-template/blob/master/readme.md#template-syntax) + +The characters `:`, `*`, `(`, and `)` have special meanings. + +`:` indicates the following segment is the name of a variable +`*` indicates the following segment is the splat/glob +`(` starts an optional segment +`)` ends an optional segment + +additionally `/` and `.` will start a new segment. + +##### Static Segments + + "/foo/bar.baz" + ^ ^ ^ + | | Starts a segment, matching ".baz" + | | + | Starts a segment, matching "/bar" + | + Starts a segment, matching "/foo" + +##### Variables + + "/foo/:bar.baz" + ^ ^ ^ + | | Starts a new segment, that matches ".baz" + | | + | Matches anything up to the start of the next segment, with the value + | being stored in the "bar" parameter of the returned match object + | + Starts a segment, matching "/foo" + +##### Splat/Glob + + "/foo/*bar" + ^ ^ + | Matches any number of segments, the values being stored as an array + | in the "bar" parameter of the returned match object + | + Starts a segment, matching "/foo" + +###### Anonymous Splat/Glob + + "/foo/*" + ^ ^ + | Matches any number of segments, the values will not appear in the + | returned match object + | + Starts a segment, matching "/foo" + +##### Optional Segments + + "/foo(/baz)/baz" + ^ ^ ^^ + | | |Starts a new segment, that matches "/baz" + | | | + | | Ends the optional segment + | | + | Starts an optional segment, this segment need not be in the path being + | matched for the match to be successful + | + Starts a segment, matching "/foo" + +### More examples + +Examples of `template` usage cases you can find in our +[test files](../test/image-data.js). + +## S3 event notification limitations + +S3 event notifications are limited to filter file events only by predefined +`prefix` and `suffix`. This could be problematic in situation where you decide +to store output files in the same directory structure as the input image. This +could cause S3 to fire new event notification for each output image saved in +that path. In extreme case this could lead to situation where Lambda +functions are executed in never ending loop. But, we are prepared to prevent +such incidents, or maybe I should rather say, we are prepared to minimise the +potential damage. + +Each processed file is stored with additional +`Metadata: { img-processed: true }`. Also, each input file that we process is +checked against this `flag`, and if it's present, we will stop the processing +flow with `"Object was already processed."` error message. diff --git a/lib/ImageArchiver.js b/lib/ImageArchiver.js index c697af2..baed4d5 100644 --- a/lib/ImageArchiver.js +++ b/lib/ImageArchiver.js @@ -30,7 +30,12 @@ class ImageArchiver { resolve( new ImageData( - image.combineWithDirectory(option.directory, option.prefix, option.suffix), + image.combineWithDirectory({ + directory: option.directory, + template: option.template, + prefix: option.prefix, + suffix: option.suffix + }), option.bucket || image.bucketName, image.data, image.headers, diff --git a/lib/ImageData.js b/lib/ImageData.js index fdd6abe..d949f62 100644 --- a/lib/ImageData.js +++ b/lib/ImageData.js @@ -1,7 +1,8 @@ "use strict"; -const path = require("path"); -const imageType = require("image-type"); +const path = require("path"); +const imageType = require("image-type"); +const PathTemplate = require("path-template"); class ImageType { @@ -156,11 +157,27 @@ class ImageData { * @param String fileSuffix (from options) * @return String */ - combineWithDirectory(directory, filePrefix, fileSuffix) { - const prefix = filePrefix || ""; - const suffix = fileSuffix || ""; + combineWithDirectory(output) { + const prefix = output.prefix || ""; + const suffix = output.suffix || ""; const fileName = path.parse(this.baseName).name; const extension = "." + this.type.ext; + + const template = output.template; + if ( typeof template === "object" && template.pattern ) { + const inputTemplate = PathTemplate.parse(template.pattern); + const outputTemplate = PathTemplate.parse(template.output || ""); + + const match = PathTemplate.match(inputTemplate, this.dirName); + if ( match ) { + const outputPath = PathTemplate.format(outputTemplate, match); + return path.join(outputPath, prefix + fileName + suffix + extension); + } else { + console.log( "Directory " + this.dirName + " didn't match template " + template.pattern ); + } + } + + const directory = output.directory; if ( typeof directory === "string" ) { // ./X , ../X , . , .. if ( directory.match(/^\.\.?\//) || directory.match(/^\.\.?$/) ) { @@ -168,9 +185,9 @@ class ImageData { } else { return path.join(directory, prefix + fileName + suffix + extension); } - } else { - return path.join(this.dirName, prefix + fileName + suffix + extension); } + + return path.join(this.dirName, prefix + fileName + suffix + extension); } } diff --git a/lib/ImageReducer.js b/lib/ImageReducer.js index e81bf7a..b33e6d2 100644 --- a/lib/ImageReducer.js +++ b/lib/ImageReducer.js @@ -38,7 +38,12 @@ class ImageReducer { return chain.pipes(streams).run() .then((buffer) => { return new ImageData( - image.combineWithDirectory(option.directory, option.prefix, option.suffix), + image.combineWithDirectory({ + directory: option.directory, + template: option.template, + prefix: option.prefix, + suffix: option.suffix + }), option.bucket || image.bucketName, buffer, image.headers, diff --git a/package.json b/package.json index f3d6ed2..1e8ab58 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ "dependencies": { "aws-sdk": "^2.24.0", "gm": "^1.23.0", - "image-type": "^3.0.0" + "image-type": "^3.0.0", + "path-template": "0.0.0" }, "devDependencies": { "ava": "^0.18.2", diff --git a/test/image-data.js b/test/image-data.js index 2386ece..09e036c 100644 --- a/test/image-data.js +++ b/test/image-data.js @@ -3,39 +3,108 @@ const ImageData = require("../lib/ImageData"); const test = require("ava"); -test("ImageData combineWithDirectory Test", t => { - const image = new ImageData("a/b/c/key.png", "bucket", "data", {}); +let image; - // No directory - t.is(image.combineWithDirectory(undefined), "a/b/c/key.png"); +test.before(t => { + image = new ImageData("a/b/c/key.png", "bucket", "data", {}); +}); + +test("Build output path when directory is undefined", t => { + t.is(image.combineWithDirectory({}), "a/b/c/key.png"); +}); + +test("Build output path when directory is empty", t => { + t.is(image.combineWithDirectory({directory: ""}), "key.png"); +}); + +test("Build output path when directory is relative deeper", t => { + t.is(image.combineWithDirectory({directory: "./d"}), "a/b/c/d/key.png"); +}); + +test("Build output path when directory is relative deeper - 2nd level", t => { + t.is(image.combineWithDirectory({directory: "./d/e"}), "a/b/c/d/e/key.png"); +}); + +test("Build output path when directory is relative backward", t => { + t.is(image.combineWithDirectory({directory: ".."}), "a/b/key.png"); +}); + +test("Build output path when directory is relative backward with new subdirectory branch", t => { + t.is(image.combineWithDirectory({directory: "../d"}), "a/b/d/key.png"); +}); + +test("Build output path when directory is absolute", t => { + t.is(image.combineWithDirectory({directory: "d"}), "d/key.png"); +}); + +test("Build output path when directory is absolute - 2nd level", t => { + t.is(image.combineWithDirectory({directory: "d/e"}), "d/e/key.png"); +}); + +test("Build output path with prefix", t => { + t.is(image.combineWithDirectory({directory: "d/e", prefix: "prefix-"}), "d/e/prefix-key.png"); +}); + +test("Build output path with suffix", t => { + t.is(image.combineWithDirectory({directory: "d/e", suffix: "-suffix"}), "d/e/key-suffix.png"); +}); + +test("Build output path with prefix and suffix", t => { + t.is(image.combineWithDirectory({directory: "d/e", prefix: "prefix-", suffix: "_suffix"}), "d/e/prefix-key_suffix.png"); +}); + +test("[path-template] Build output path when template is an empty object", t => { + t.is(image.combineWithDirectory({}), "a/b/c/key.png"); +}); + +test("[path-template] Build output path when template is an empty map", t => { + t.is(image.combineWithDirectory({template: {}}), "a/b/c/key.png"); +}); + +test("[path-template] Build output path when template is an map with pattern and output keys empty", t => { + t.is(image.combineWithDirectory({template: {pattern: "", output: ""}}), "a/b/c/key.png"); +}); + +test("[path-template] Build output path when template replace whole directory", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: ""}}), "key.png"); +}); - // Empty directory - t.is(image.combineWithDirectory(""), "key.png"); +test("[path-template] Build output path when template adds subdirectory", t => { + t.is(image.combineWithDirectory({template: {pattern: "*path", output: "*path/d"}}), "a/b/c/d/key.png"); +}); - // Relative directory - t.is(image.combineWithDirectory("./d"), "a/b/c/d/key.png"); +test("[path-template] Build output path when template adds subdirectory - 2nd level", t => { + t.is(image.combineWithDirectory({template: {pattern: "*path", output: "*path/d/e"}}), "a/b/c/d/e/key.png"); +}); - // Internal directory - t.is(image.combineWithDirectory("./d/e"), "a/b/c/d/e/key.png"); +test("[path-template] Build output path when template removes top subdirectory", t => { + t.is(image.combineWithDirectory({template: {pattern: "*path/c", output: "*path"}}), "a/b/key.png"); +}); - // External directory - t.is(image.combineWithDirectory(".."), "a/b/key.png"); +test("[path-template] Build output path when template replaces top subdirectory with new one", t => { + t.is(image.combineWithDirectory({template: {pattern: "*path/c", output: "*path/d"}}), "a/b/d/key.png"); +}); - // External internal directory - t.is(image.combineWithDirectory("../d"), "a/b/d/key.png"); +test("[path-template] Build output path when template replaces old path with new absolute one", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: "d"}}), "d/key.png"); +}); - // Root directory - t.is(image.combineWithDirectory("d"), "d/key.png"); +test("[path-template] Build output path when template replaces old path with new absolute one - 2nd level", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: "d/e"}}), "d/e/key.png"); +}); - // Root internal directory - t.is(image.combineWithDirectory("d/e"), "d/e/key.png"); +test("[path-template] Build output path when template didn't match base directory", t => { + t.is(image.combineWithDirectory({template: {pattern: "x/:something", output: "d/e"}}), "a/b/c/key.png"); +}); - // With prefix - t.is(image.combineWithDirectory("d/e", "prefix-"), "d/e/prefix-key.png"); +test("[path-template] Build output path with template and prefix", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: "d/e"}, prefix: "prefix-"}), "d/e/prefix-key.png"); +}); - // With suffix - t.is(image.combineWithDirectory("d/e", "", "-suffix"), "d/e/key-suffix.png"); +test("[path-template] Build output path with template and suffix", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: "d/e"}, suffix: "-suffix"}), "d/e/key-suffix.png"); +}); - // With prefix and suffix - t.is(image.combineWithDirectory("d/e", "prefix-", "_suffix"), "d/e/prefix-key_suffix.png"); +test("[path-template] Build output path with template, prefix and suffix", t => { + t.is(image.combineWithDirectory({template: {pattern: "*", output: "d/e"}, prefix: "prefix-", suffix: "_suffix"}), "d/e/prefix-key_suffix.png"); });