From 83de9e0ed0960aaa7e7f46e6886b262c876d5795 Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 12 Apr 2022 18:30:44 +0300 Subject: [PATCH 01/19] egld network created + egld output files in same folder + custom output paths for each network --- constants/network.js | 17 +++++++++++++++-- src/config.js | 2 +- src/main.js | 18 +++++++++--------- utils/preview.js | 4 ++-- utils/rarity.js | 2 +- utils/update_info.js | 6 +++--- 6 files changed, 31 insertions(+), 18 deletions(-) diff --git a/constants/network.js b/constants/network.js index bd962822e..d2520bab0 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,6 +1,19 @@ const NETWORK = { - eth: "eth", - sol: "sol", + eth: { + name: "eth", + jsonDirPrefix: "json/", + mediaDirPrefix: "media/", + }, + sol: { + name: "sol", + jsonDirPrefix: "json/", + mediaDirPrefix: "media/", + }, + egld: { + name: "egld", + jsonDirPrefix: "", + mediaDirPrefix: "", + }, }; module.exports = { diff --git a/src/config.js b/src/config.js index c46d867a0..17858dfe6 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.eth; +const network = NETWORK.egld; // General metadata for Ethereum const namePrefix = "Your Collection"; diff --git a/src/main.js b/src/main.js index e9c08dcf2..5d38274eb 100644 --- a/src/main.js +++ b/src/main.js @@ -38,11 +38,11 @@ const buildSetup = () => { fs.rmdirSync(buildDir, { recursive: true }); } fs.mkdirSync(buildDir); - fs.mkdirSync(`${buildDir}/json`); - fs.mkdirSync(`${buildDir}/images`); - if (gif.export) { - fs.mkdirSync(`${buildDir}/gifs`); - } + + if (network.jsonDirPrefix) + fs.mkdirSync(`${buildDir}/${network.jsonDirPrefix}`); + if (network.mediaDirPrefix) + fs.mkdirSync(`${buildDir}/${network.mediaDirPrefix}`); }; const getRarityWeight = (_str) => { @@ -112,7 +112,7 @@ const layersSetup = (layersOrder) => { const saveImage = (_editionCount) => { fs.writeFileSync( - `${buildDir}/images/${_editionCount}.png`, + `${buildDir}/${network.mediaDirPrefix}$${_editionCount}.png`, canvas.toBuffer("image/png") ); }; @@ -304,7 +304,7 @@ const createDna = (_layers) => { }; const writeMetaData = (_data) => { - fs.writeFileSync(`${buildDir}/json/_metadata.json`, _data); + fs.writeFileSync(`${buildDir}/${network.jsonDirPrefix}_metadata.json`, _data); }; const saveMetaDataSingleFile = (_editionCount) => { @@ -315,7 +315,7 @@ const saveMetaDataSingleFile = (_editionCount) => { ) : null; fs.writeFileSync( - `${buildDir}/json/${_editionCount}.json`, + `${buildDir}/${network.jsonDirPrefix}${_editionCount}.json`, JSON.stringify(metadata, null, 2) ); }; @@ -375,7 +375,7 @@ const startCreating = async () => { hashlipsGiffer = new HashlipsGiffer( canvas, ctx, - `${buildDir}/gifs/${abstractedIndexes[0]}.gif`, + `${buildDir}/${network.mediaDirPrefix}${abstractedIndexes[0]}.gif`, gif.repeat, gif.quality, gif.delay diff --git a/utils/preview.js b/utils/preview.js index 5765e6c81..5b05649e9 100644 --- a/utils/preview.js +++ b/utils/preview.js @@ -6,7 +6,7 @@ const buildDir = `${basePath}/build`; const { preview } = require(`${basePath}/src/config.js`); // read json data -const rawdata = fs.readFileSync(`${basePath}/build/json/_metadata.json`); +const rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); const metadataList = JSON.parse(rawdata); const saveProjectPreviewImage = async (_data) => { @@ -32,7 +32,7 @@ const saveProjectPreviewImage = async (_data) => { // Don't want to rely on "edition" for assuming index for (let index = 0; index < _data.length; index++) { const nft = _data[index]; - await loadImage(`${buildDir}/images/${nft.edition}.png`).then((image) => { + await loadImage(`${buildDir}/${network.mediaDirPrefix}$${nft.edition}.png`).then((image) => { previewCtx.drawImage( image, thumbWidth * (index % thumbPerRow), diff --git a/utils/rarity.js b/utils/rarity.js index ac577aaf3..ef1e5858b 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -7,7 +7,7 @@ const { layerConfigurations } = require(`${basePath}/src/config.js`); const { getElements } = require("../src/main.js"); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/json/_metadata.json`); +let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); let data = JSON.parse(rawdata); let editionSize = data.length; diff --git a/utils/update_info.js b/utils/update_info.js index f51788631..9b24e8897 100644 --- a/utils/update_info.js +++ b/utils/update_info.js @@ -11,7 +11,7 @@ const { } = require(`${basePath}/src/config.js`); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/json/_metadata.json`); +let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); let data = JSON.parse(rawdata); data.forEach((item) => { @@ -25,13 +25,13 @@ data.forEach((item) => { item.image = `${baseUri}/${item.edition}.png`; } fs.writeFileSync( - `${basePath}/build/json/${item.edition}.json`, + `${basePath}/build/${network.jsonDirPrefix}${item.edition}.json`, JSON.stringify(item, null, 2) ); }); fs.writeFileSync( - `${basePath}/build/json/_metadata.json`, + `${basePath}/build/${network.jsonDirPrefix}_metadata.json`, JSON.stringify(data, null, 2) ); From 2c0fe73136b306532670b8020c73fa790bcc00ba Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 12 Apr 2022 22:06:56 +0300 Subject: [PATCH 02/19] terrain prepared. now we can calculate rarities --- README.md | 4 ++-- constants/network.js | 14 ++++++++++++++ src/main.js | 14 +++++++++++--- utils/generate_metadata.js | 7 ++++--- utils/pixelate.js | 2 +- utils/preview.js | 2 +- utils/preview_gif.js | 2 +- utils/rarity.js | 2 +- utils/update_info.js | 4 ++-- 9 files changed, 37 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 98a393c2e..2ac0970b7 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ const MODE = { }; ``` -When you are ready, run the following command and your outputted art will be in the `build/images` directory and the json in the `build/json` directory: +When you are ready, run the following command and your outputted art will be in the `build${network.mediaDirPrefix}` directory and the json in the `build${network.jsonDirPrefix}` directory: ```sh npm run build @@ -190,7 +190,7 @@ or node index.js ``` -The program will output all the images in the `build/images` directory along with the metadata files in the `build/json` directory. Each collection will have a `_metadata.json` file that consists of all the metadata in the collection inside the `build/json` directory. The `build/json` folder also will contain all the single json files that represent each image file. The single json file of a image will look something like this: +The program will output all the images in the `build${network.mediaDirPrefix}` directory along with the metadata files in the `build${network.jsonDirPrefix}` directory. Each collection will have a `${network.metadataFileName}` file that consists of all the metadata in the collection inside the `build${network.jsonDirPrefix}` directory. The `build${network.jsonDirPrefix}` folder also will contain all the single json files that represent each image file. The single json file of a image will look something like this: ```json { diff --git a/constants/network.js b/constants/network.js index d2520bab0..667120956 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,21 +1,35 @@ +const metadataTypes = { + // metadata file will contain all individual metadata files (e.g. eth$, sol$) + basic: 0, + // metadata file will contain only rarities (e.g. egld$) + rarities: 1, +}; + const NETWORK = { eth: { name: "eth", jsonDirPrefix: "json/", mediaDirPrefix: "media/", + metadataFileName: "_metadata.json", + metadataType: metadataTypes.basic, }, sol: { name: "sol", jsonDirPrefix: "json/", mediaDirPrefix: "media/", + metadataFileName: "_metadata.json", + metadataType: metadataTypes.basic, }, egld: { name: "egld", jsonDirPrefix: "", mediaDirPrefix: "", + metadataFileName: "_metadata.json", + metadataType: metadataTypes.rarities, }, }; module.exports = { NETWORK, + metadataTypes, }; diff --git a/src/main.js b/src/main.js index 5d38274eb..4a01e413f 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,5 @@ const basePath = process.cwd(); -const { NETWORK } = require(`${basePath}/constants/network.js`); +const { NETWORK, metadataTypes } = require(`${basePath}/constants/network.js`); const fs = require("fs"); const sha1 = require(`${basePath}/node_modules/sha1`); const { createCanvas, loadImage } = require(`${basePath}/node_modules/canvas`); @@ -304,7 +304,10 @@ const createDna = (_layers) => { }; const writeMetaData = (_data) => { - fs.writeFileSync(`${buildDir}/${network.jsonDirPrefix}_metadata.json`, _data); + fs.writeFileSync( + `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, + _data + ); }; const saveMetaDataSingleFile = (_editionCount) => { @@ -426,7 +429,12 @@ const startCreating = async () => { } layerConfigIndex++; } - writeMetaData(JSON.stringify(metadataList, null, 2)); + + if (network.metadataType == metadataTypes.basic) + writeMetaData(JSON.stringify(metadataList, null, 2)); + else if (network.metadataType == metadataTypes.rarities) { + // calculate rarities and generate metadata file + } }; module.exports = { startCreating, buildSetup, getElements }; diff --git a/utils/generate_metadata.js b/utils/generate_metadata.js index 21d0a7ca9..458490ed9 100644 --- a/utils/generate_metadata.js +++ b/utils/generate_metadata.js @@ -2,8 +2,8 @@ const fs = require("fs"); const path = require("path"); const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); -const buildDir = `${basePath}/build/json`; -const inputDir = `${basePath}/build/images`; +const buildDir = `${basePath}/build/${network.jsonDirPath}`; +const inputDir = `${basePath}/build/${network.jsonDirPath}`; const { format, namePrefix, @@ -11,6 +11,7 @@ const { baseUri, } = require(`${basePath}/src/config.js`); const console = require("console"); +const { network } = require("../src/config"); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); const metadataList = []; @@ -153,7 +154,7 @@ const saveMetadata = (_loadedImageObject) => { }; const writeMetaData = (_data) => { - fs.writeFileSync(`${buildDir}/_metadata.json`, _data); + fs.writeFileSync(`${buildDir}/${network.metadataFileName}`, _data); }; const startCreating = async () => { diff --git a/utils/pixelate.js b/utils/pixelate.js index 5fa3be377..e686aa68c 100644 --- a/utils/pixelate.js +++ b/utils/pixelate.js @@ -3,7 +3,7 @@ const path = require("path"); const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); const buildDir = `${basePath}/build/pixel_images`; -const inputDir = `${basePath}/build/images`; +const inputDir = `${basePath}/build/${network.mediaDirPrefix}`; const { format, pixelFormat } = require(`${basePath}/src/config.js`); const console = require("console"); const canvas = createCanvas(format.width, format.height); diff --git a/utils/preview.js b/utils/preview.js index 5b05649e9..8167366ce 100644 --- a/utils/preview.js +++ b/utils/preview.js @@ -6,7 +6,7 @@ const buildDir = `${basePath}/build`; const { preview } = require(`${basePath}/src/config.js`); // read json data -const rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); +const rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); const metadataList = JSON.parse(rawdata); const saveProjectPreviewImage = async (_data) => { diff --git a/utils/preview_gif.js b/utils/preview_gif.js index 0b0c339ba..d52a0535a 100644 --- a/utils/preview_gif.js +++ b/utils/preview_gif.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const fs = require("fs"); const { createCanvas, loadImage } = require("canvas"); const buildDir = `${basePath}/build`; -const imageDir = `${buildDir}/images`; +const imageDir = `${buildDir}/${network.mediaDirPrefix}`; const { format, preview_gif } = require(`${basePath}/src/config.js`); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); diff --git a/utils/rarity.js b/utils/rarity.js index ef1e5858b..48e6c82b1 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -7,7 +7,7 @@ const { layerConfigurations } = require(`${basePath}/src/config.js`); const { getElements } = require("../src/main.js"); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); +let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); let data = JSON.parse(rawdata); let editionSize = data.length; diff --git a/utils/update_info.js b/utils/update_info.js index 9b24e8897..308767654 100644 --- a/utils/update_info.js +++ b/utils/update_info.js @@ -11,7 +11,7 @@ const { } = require(`${basePath}/src/config.js`); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}_metadata.json`); +let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); let data = JSON.parse(rawdata); data.forEach((item) => { @@ -31,7 +31,7 @@ data.forEach((item) => { }); fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix}_metadata.json`, + `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`, JSON.stringify(data, null, 2) ); From 1b48d5347b033cc4f380c1dad218e44d70f8b6d1 Mon Sep 17 00:00:00 2001 From: johnykes Date: Fri, 15 Apr 2022 00:04:51 +0300 Subject: [PATCH 03/19] network customisation functionality, egld network config added, rarity functionality added using xterr/nft-generator`s logic + a little fix --- constants/network.js | 17 ++-- src/config.js | 5 +- src/main.js | 204 ++++++++++++++++++++++++++++++++++--------- utils/rarity.js | 2 +- utils/update_info.js | 18 ++-- 5 files changed, 190 insertions(+), 56 deletions(-) diff --git a/constants/network.js b/constants/network.js index 667120956..97248bfee 100644 --- a/constants/network.js +++ b/constants/network.js @@ -6,8 +6,17 @@ const metadataTypes = { }; const NETWORK = { + egld: { + name: "egld", + startIdx: 1, + jsonDirPrefix: "", + mediaDirPrefix: "", + metadataFileName: "_metadata.json", + metadataType: metadataTypes.rarities, + }, eth: { name: "eth", + startIdx: 1, jsonDirPrefix: "json/", mediaDirPrefix: "media/", metadataFileName: "_metadata.json", @@ -15,18 +24,12 @@ const NETWORK = { }, sol: { name: "sol", + startIdx: 0, jsonDirPrefix: "json/", mediaDirPrefix: "media/", metadataFileName: "_metadata.json", metadataType: metadataTypes.basic, }, - egld: { - name: "egld", - jsonDirPrefix: "", - mediaDirPrefix: "", - metadataFileName: "_metadata.json", - metadataType: metadataTypes.rarities, - }, }; module.exports = { diff --git a/src/config.js b/src/config.js index 17858dfe6..576528ffd 100644 --- a/src/config.js +++ b/src/config.js @@ -4,11 +4,14 @@ const { NETWORK } = require(`${basePath}/constants/network.js`); const network = NETWORK.egld; -// General metadata for Ethereum +// General metadata const namePrefix = "Your Collection"; const description = "Remember to replace this description"; + +// Ethereum metadata const baseUri = "ipfs://NewUriToReplace"; +// Solana metadata const solanaMetadata = { symbol: "YC", seller_fee_basis_points: 1000, // Define how much % you want from secondary market sales 1000 = 10% diff --git a/src/main.js b/src/main.js index 4a01e413f..65c2694b9 100644 --- a/src/main.js +++ b/src/main.js @@ -28,6 +28,7 @@ ctx.imageSmoothingEnabled = format.smoothing; var metadataList = []; var attributesList = []; var dnaList = new Set(); +let rarityObject = {}; const DNA_DELIMITER = "-"; const HashlipsGiffer = require(`${basePath}/modules/HashlipsGiffer.js`); @@ -129,48 +130,139 @@ const drawBackground = () => { }; const addMetadata = (_dna, _edition) => { - let dateTime = Date.now(); - let tempMetadata = { - name: `${namePrefix} #${_edition}`, - description: description, - image: `${baseUri}/${_edition}.png`, - dna: sha1(_dna), - edition: _edition, - date: dateTime, - ...extraMetadata, - attributes: attributesList, - compiler: "HashLips Art Engine", - }; - if (network == NETWORK.sol) { - tempMetadata = { - //Added metadata for solana - name: tempMetadata.name, - symbol: solanaMetadata.symbol, - description: tempMetadata.description, - //Added metadata for solana - seller_fee_basis_points: solanaMetadata.seller_fee_basis_points, - image: `${_edition}.png`, - //Added metadata for solana - external_url: solanaMetadata.external_url, - edition: _edition, - ...extraMetadata, - attributes: tempMetadata.attributes, - properties: { - files: [ - { - uri: `${_edition}.png`, - type: "image/png", - }, - ], - category: "image", - creators: solanaMetadata.creators, - }, - }; + let tempMetadata = {}; + + switch (network) { + case NETWORK.egld: { + tempMetadata = { + description: description, + dna: sha1(_dna), + ...extraMetadata, + attributes: attributesList, + rarities: {}, + compiler: "HashLips Art Engine", + }; + } + case NETWORK.eth: { + tempMetadata = { + name: `${namePrefix} #${_edition}`, + description: description, + image: `${baseUri}/${_edition}.png`, + dna: sha1(_dna), + edition: _edition, + date: Date.now(), + attributes: attributesList, + ...extraMetadata, + compiler: "HashLips Art Engine", + }; + break; + } + case NETWORK.sol: { + tempMetadata = { + name: `${namePrefix} #${_edition}`, + symbol: solanaMetadata.symbol, + description: tempMetadata.description, + seller_fee_basis_points: solanaMetadata.seller_fee_basis_points, + image: `${_edition}.png`, + external_url: solanaMetadata.external_url, + edition: _edition, + ...extraMetadata, + attributes: tempMetadata.attributes, + properties: { + files: [ + { + uri: `${_edition}.png`, + type: "image/png", + }, + ], + category: "image", + creators: solanaMetadata.creators, + }, + }; + break; + } } + metadataList.push(tempMetadata); attributesList = []; }; +const addRarityMetadata = () => { + // currently using https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts logic + // ps: the only difference is that the attributeRarityNormed is calculated only once (in case there are NFTs with less/more attributes than others) + // todo: add multiple rarity implementations + + let traitOccurances = []; + let totalAttributesCnt = 0; + + // count occurrences for all traits/layers & attributes/assets + metadataList.forEach((item) => { + item.attributes.forEach((a) => { + if (rarityObject[a.trait_type] != null) { + if (rarityObject[a.trait_type][a.value] != null) { + rarityObject[a.trait_type][a.value].attributeOccurrence++; + } else { + rarityObject[a.trait_type][a.value] = { attributeOccurrence: 1 }; + totalAttributesCnt++; + } + + traitOccurances[a.trait_type]++; + } else { + rarityObject[a.trait_type] = { [a.value]: { attributeOccurrence: 1 } }; + traitOccurances[a.trait_type] = 1; + totalAttributesCnt++; + } + }); + }); + + const totalLayersCnt = Object.keys(traitOccurances).length; + const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; + + // calculate rarity for all traits/layers & attributes/assets + Object.entries(rarityObject).forEach((entry) => { + const layer = entry[0]; + const assets = entry[1]; + + Object.entries(assets).forEach((asset) => { + // trait/layer related + asset[1].traitOccurance = traitOccurances[layer]; + asset[1].traitOccurancePercentage = + (metadataList.length / asset[1].traitOccurance) * 100; + asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; + + // attribute/asset related + asset[1].attributeFrequency = + asset[1].attributeOccurrence / metadataList.length; + asset[1].attributeRarity = + metadataList.length / asset[1].attributeOccurrence; + // ugly fix for the original implementation + asset[1].attributeRarityNormed = + asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); + }); + }); + + // calculate rarity for each item/NFT + metadataList.forEach((item) => { + item.rarity = { + avgRarity: 0, + statRarity: 1, + rarityScore: 0, + rarityScoreNormed: 0, + usedTraitsCount: item.attributes.length, + }; + + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + // original buggy implementation (not ok if different nr. of attributes) + //attributeData.attributeRarityNormed = attributeData.attributeRarity * avgAttributesPerTrait / item.attributes.length; + item.rarity.avgRarity += attributeData.attributeFrequency; + item.rarity.statRarity *= attributeData.attributeFrequency; + item.rarity.rarityScore += attributeData.attributeRarity; + item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + }); + }); +}; + const addAttributes = (_element) => { let selectedElement = _element.layer.selectedElement; attributesList.push({ @@ -310,7 +402,7 @@ const writeMetaData = (_data) => { ); }; -const saveMetaDataSingleFile = (_editionCount) => { +/*const saveMetaDataSingleFile = (_editionCount) => { let metadata = metadataList.find((meta) => meta.edition == _editionCount); debugLogs ? console.log( @@ -321,6 +413,26 @@ const saveMetaDataSingleFile = (_editionCount) => { `${buildDir}/${network.jsonDirPrefix}${_editionCount}.json`, JSON.stringify(metadata, null, 2) ); +};*/ + +const saveIndividualMetadataFiles = (abstractedIndexes) => { + let idx = 0; + metadataList.forEach((item) => { + debugLogs + ? console.log( + `Writing metadata for ${ + item.edition || abstractedIndexes[idx] + }: ${JSON.stringify(item)}` + ) + : null; + fs.writeFileSync( + `${buildDir}/${network.jsonDirPrefix}${ + item.edition || abstractedIndexes[idx] + }.json`, + JSON.stringify(item, null, 2) + ); + idx++; + }); }; function shuffle(array) { @@ -342,8 +454,9 @@ const startCreating = async () => { let editionCount = 1; let failedCount = 0; let abstractedIndexes = []; + let abstractedIndexesBackup = []; for ( - let i = network == NETWORK.sol ? 0 : 1; + let i = network.startIdx; i <= layerConfigurations[layerConfigurations.length - 1].growEditionSizeTo; i++ ) { @@ -352,6 +465,7 @@ const startCreating = async () => { if (shuffleLayerConfigurations) { abstractedIndexes = shuffle(abstractedIndexes); } + abstractedIndexesBackup = [...abstractedIndexes]; debugLogs ? console.log("Editions left to create: ", abstractedIndexes) : null; @@ -406,7 +520,6 @@ const startCreating = async () => { : null; saveImage(abstractedIndexes[0]); addMetadata(newDna, abstractedIndexes[0]); - saveMetaDataSingleFile(abstractedIndexes[0]); console.log( `Created edition: ${abstractedIndexes[0]}, with DNA: ${sha1( newDna @@ -430,10 +543,19 @@ const startCreating = async () => { layerConfigIndex++; } + // build rarity (if needed) + if ((network.metadataType = metadataTypes.rarities)) { + addRarityMetadata(); + } + + // save individual metadata files + saveIndividualMetadataFiles(abstractedIndexesBackup); + + // save metadata.json if (network.metadataType == metadataTypes.basic) writeMetaData(JSON.stringify(metadataList, null, 2)); else if (network.metadataType == metadataTypes.rarities) { - // calculate rarities and generate metadata file + writeMetaData(JSON.stringify(rarityObject, null, 2)); } }; diff --git a/utils/rarity.js b/utils/rarity.js index 48e6c82b1..fc18fcaa5 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -59,7 +59,7 @@ data.forEach((element) => { }); }); -// convert occurrences to occurence string +// convert occurrences to occurrences string for (var layer in rarityData) { for (var attribute in rarityData[layer]) { // get chance diff --git a/utils/update_info.js b/utils/update_info.js index 308767654..a7875fc0f 100644 --- a/utils/update_info.js +++ b/utils/update_info.js @@ -11,19 +11,25 @@ const { } = require(`${basePath}/src/config.js`); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); +let rawdata = fs.readFileSync( + `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` +); let data = JSON.parse(rawdata); data.forEach((item) => { + // general metadata + item.description = description; + + // custom metadata + if (network == NETWORK.eth) { + item.image = `${baseUri}/${item.edition}.png`; + item.name = `${namePrefix} #${item.edition}`; + } if (network == NETWORK.sol) { item.name = `${namePrefix} #${item.edition}`; - item.description = description; item.creators = solanaMetadata.creators; - } else { - item.name = `${namePrefix} #${item.edition}`; - item.description = description; - item.image = `${baseUri}/${item.edition}.png`; } + fs.writeFileSync( `${basePath}/build/${network.jsonDirPrefix}${item.edition}.json`, JSON.stringify(item, null, 2) From 0235a82c70d3920501dac7b64ee2c7c74676ad5a Mon Sep 17 00:00:00 2001 From: johnykes Date: Fri, 15 Apr 2022 09:29:33 +0300 Subject: [PATCH 04/19] mediaFilePrefix customization and small bug fix --- constants/network.js | 7 +++++-- src/config.js | 2 +- src/main.js | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/constants/network.js b/constants/network.js index 97248bfee..82d07f896 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,7 +1,7 @@ const metadataTypes = { - // metadata file will contain all individual metadata files (e.g. eth$, sol$) + // metadata file will contain all individual metadata files (common for eth$, sol$) basic: 0, - // metadata file will contain only rarities (e.g. egld$) + // metadata file will contain only rarities (common egld$) rarities: 1, }; @@ -11,6 +11,7 @@ const NETWORK = { startIdx: 1, jsonDirPrefix: "", mediaDirPrefix: "", + mediaFilePrefix: "", metadataFileName: "_metadata.json", metadataType: metadataTypes.rarities, }, @@ -19,6 +20,7 @@ const NETWORK = { startIdx: 1, jsonDirPrefix: "json/", mediaDirPrefix: "media/", + mediaFilePrefix: "$", metadataFileName: "_metadata.json", metadataType: metadataTypes.basic, }, @@ -27,6 +29,7 @@ const NETWORK = { startIdx: 0, jsonDirPrefix: "json/", mediaDirPrefix: "media/", + mediaFilePrefix: "$", metadataFileName: "_metadata.json", metadataType: metadataTypes.basic, }, diff --git a/src/config.js b/src/config.js index 576528ffd..dff85151e 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.egld; +const network = NETWORK.eth; // General metadata const namePrefix = "Your Collection"; diff --git a/src/main.js b/src/main.js index 65c2694b9..d591ab5ae 100644 --- a/src/main.js +++ b/src/main.js @@ -113,7 +113,7 @@ const layersSetup = (layersOrder) => { const saveImage = (_editionCount) => { fs.writeFileSync( - `${buildDir}/${network.mediaDirPrefix}$${_editionCount}.png`, + `${buildDir}/${network.mediaDirPrefix}${network.mediaFilePrefix}${_editionCount}.png`, canvas.toBuffer("image/png") ); }; @@ -191,7 +191,7 @@ const addRarityMetadata = () => { // currently using https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts logic // ps: the only difference is that the attributeRarityNormed is calculated only once (in case there are NFTs with less/more attributes than others) // todo: add multiple rarity implementations - + let traitOccurances = []; let totalAttributesCnt = 0; @@ -544,7 +544,7 @@ const startCreating = async () => { } // build rarity (if needed) - if ((network.metadataType = metadataTypes.rarities)) { + if ((network.metadataType == metadataTypes.rarities)) { addRarityMetadata(); } From 5e2be9fa1edcbf27bd3e0958bc917943d988031e Mon Sep 17 00:00:00 2001 From: johnykes Date: Fri, 15 Apr 2022 09:45:37 +0300 Subject: [PATCH 05/19] easier network customization --- constants/network.js | 3 --- src/config.js | 2 +- src/main.js | 16 +++++++++------- utils/pixelate.js | 2 +- utils/preview.js | 8 ++++++-- utils/preview_gif.js | 2 +- utils/rarity.js | 2 +- utils/update_info.js | 6 +++--- 8 files changed, 22 insertions(+), 19 deletions(-) diff --git a/constants/network.js b/constants/network.js index 82d07f896..e45be9e64 100644 --- a/constants/network.js +++ b/constants/network.js @@ -9,9 +9,6 @@ const NETWORK = { egld: { name: "egld", startIdx: 1, - jsonDirPrefix: "", - mediaDirPrefix: "", - mediaFilePrefix: "", metadataFileName: "_metadata.json", metadataType: metadataTypes.rarities, }, diff --git a/src/config.js b/src/config.js index dff85151e..576528ffd 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.eth; +const network = NETWORK.egld; // General metadata const namePrefix = "Your Collection"; diff --git a/src/main.js b/src/main.js index d591ab5ae..2b0ca9f8e 100644 --- a/src/main.js +++ b/src/main.js @@ -41,9 +41,9 @@ const buildSetup = () => { fs.mkdirSync(buildDir); if (network.jsonDirPrefix) - fs.mkdirSync(`${buildDir}/${network.jsonDirPrefix}`); + fs.mkdirSync(`${buildDir}/${network.jsonDirPrefix ?? ""}`); if (network.mediaDirPrefix) - fs.mkdirSync(`${buildDir}/${network.mediaDirPrefix}`); + fs.mkdirSync(`${buildDir}/${network.mediaDirPrefix ?? ""}`); }; const getRarityWeight = (_str) => { @@ -113,7 +113,9 @@ const layersSetup = (layersOrder) => { const saveImage = (_editionCount) => { fs.writeFileSync( - `${buildDir}/${network.mediaDirPrefix}${network.mediaFilePrefix}${_editionCount}.png`, + `${buildDir}/${network.mediaDirPrefix ?? ""}${ + network.mediaFilePrefix ?? "" + }${_editionCount}.png`, canvas.toBuffer("image/png") ); }; @@ -397,7 +399,7 @@ const createDna = (_layers) => { const writeMetaData = (_data) => { fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, + `${buildDir}/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`, _data ); }; @@ -426,7 +428,7 @@ const saveIndividualMetadataFiles = (abstractedIndexes) => { ) : null; fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix}${ + `${buildDir}/${network.jsonDirPrefix ?? ""}${ item.edition || abstractedIndexes[idx] }.json`, JSON.stringify(item, null, 2) @@ -492,7 +494,7 @@ const startCreating = async () => { hashlipsGiffer = new HashlipsGiffer( canvas, ctx, - `${buildDir}/${network.mediaDirPrefix}${abstractedIndexes[0]}.gif`, + `${buildDir}/${network.mediaDirPrefix ?? ""}${abstractedIndexes[0]}.gif`, gif.repeat, gif.quality, gif.delay @@ -544,7 +546,7 @@ const startCreating = async () => { } // build rarity (if needed) - if ((network.metadataType == metadataTypes.rarities)) { + if (network.metadataType == metadataTypes.rarities) { addRarityMetadata(); } diff --git a/utils/pixelate.js b/utils/pixelate.js index e686aa68c..cc3ee24cf 100644 --- a/utils/pixelate.js +++ b/utils/pixelate.js @@ -3,7 +3,7 @@ const path = require("path"); const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); const buildDir = `${basePath}/build/pixel_images`; -const inputDir = `${basePath}/build/${network.mediaDirPrefix}`; +const inputDir = `${basePath}/build/${network.mediaDirPrefix ?? ""}`; const { format, pixelFormat } = require(`${basePath}/src/config.js`); const console = require("console"); const canvas = createCanvas(format.width, format.height); diff --git a/utils/preview.js b/utils/preview.js index 8167366ce..824d623b9 100644 --- a/utils/preview.js +++ b/utils/preview.js @@ -6,7 +6,9 @@ const buildDir = `${basePath}/build`; const { preview } = require(`${basePath}/src/config.js`); // read json data -const rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); +const rawdata = fs.readFileSync( + `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}` +); const metadataList = JSON.parse(rawdata); const saveProjectPreviewImage = async (_data) => { @@ -32,7 +34,9 @@ const saveProjectPreviewImage = async (_data) => { // Don't want to rely on "edition" for assuming index for (let index = 0; index < _data.length; index++) { const nft = _data[index]; - await loadImage(`${buildDir}/${network.mediaDirPrefix}$${nft.edition}.png`).then((image) => { + await loadImage( + `${buildDir}/${network.mediaDirPrefix ?? ""}$${nft.edition}.png` + ).then((image) => { previewCtx.drawImage( image, thumbWidth * (index % thumbPerRow), diff --git a/utils/preview_gif.js b/utils/preview_gif.js index d52a0535a..078fc9f75 100644 --- a/utils/preview_gif.js +++ b/utils/preview_gif.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const fs = require("fs"); const { createCanvas, loadImage } = require("canvas"); const buildDir = `${basePath}/build`; -const imageDir = `${buildDir}/${network.mediaDirPrefix}`; +const imageDir = `${buildDir}/${network.mediaDirPrefix ?? ""}`; const { format, preview_gif } = require(`${basePath}/src/config.js`); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); diff --git a/utils/rarity.js b/utils/rarity.js index fc18fcaa5..78536dca3 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -7,7 +7,7 @@ const { layerConfigurations } = require(`${basePath}/src/config.js`); const { getElements } = require("../src/main.js"); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`); +let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`); let data = JSON.parse(rawdata); let editionSize = data.length; diff --git a/utils/update_info.js b/utils/update_info.js index a7875fc0f..72200ffd8 100644 --- a/utils/update_info.js +++ b/utils/update_info.js @@ -12,7 +12,7 @@ const { // read json data let rawdata = fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` + `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}` ); let data = JSON.parse(rawdata); @@ -31,13 +31,13 @@ data.forEach((item) => { } fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix}${item.edition}.json`, + `${basePath}/build/${network.jsonDirPrefix ?? ""}${item.edition}.json`, JSON.stringify(item, null, 2) ); }); fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}`, + `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`, JSON.stringify(data, null, 2) ); From a32d474bbd3b8cc1684c061975d8b9374ad0e083 Mon Sep 17 00:00:00 2001 From: johnykes Date: Sat, 16 Apr 2022 11:35:45 +0300 Subject: [PATCH 06/19] add Jaccard Distances rarity algorithm & add compatibility for existing scripts --- constants/network.js | 12 +++- src/main.js | 140 ++++++++++++++++++++++++++++++++++++------- utils/common.js | 41 +++++++++++++ utils/preview.js | 12 ++-- utils/rarity.js | 19 ++++-- utils/update_info.js | 27 +++++---- 6 files changed, 205 insertions(+), 46 deletions(-) create mode 100644 utils/common.js diff --git a/constants/network.js b/constants/network.js index e45be9e64..cb552aa1e 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,8 +1,14 @@ const metadataTypes = { // metadata file will contain all individual metadata files (common for eth$, sol$) + // no rarities basic: 0, - // metadata file will contain only rarities (common egld$) - rarities: 1, + + /// metadata file will contain only rarity data for traits & attributes + /// individual metadata file will also contain rarity data + // rarities calculated using Trait Rarity (less accurate) + rarities_TR: 1, + // rarities calculated using JaccardDistances (most accurate) + rarities_JD: 2, }; const NETWORK = { @@ -10,7 +16,7 @@ const NETWORK = { name: "egld", startIdx: 1, metadataFileName: "_metadata.json", - metadataType: metadataTypes.rarities, + metadataType: metadataTypes.rarities_JD, }, eth: { name: "eth", diff --git a/src/main.js b/src/main.js index 2b0ca9f8e..0c5dc1a72 100644 --- a/src/main.js +++ b/src/main.js @@ -163,13 +163,13 @@ const addMetadata = (_dna, _edition) => { tempMetadata = { name: `${namePrefix} #${_edition}`, symbol: solanaMetadata.symbol, - description: tempMetadata.description, + description: description, seller_fee_basis_points: solanaMetadata.seller_fee_basis_points, image: `${_edition}.png`, external_url: solanaMetadata.external_url, edition: _edition, ...extraMetadata, - attributes: tempMetadata.attributes, + attributes: attributesList, properties: { files: [ { @@ -244,25 +244,121 @@ const addRarityMetadata = () => { }); // calculate rarity for each item/NFT - metadataList.forEach((item) => { - item.rarity = { - avgRarity: 0, - statRarity: 1, - rarityScore: 0, - rarityScoreNormed: 0, - usedTraitsCount: item.attributes.length, - }; + if (network.metadataType == metadataTypes.rarities_TR) { + metadataList.forEach((item) => { + item.rarity = { + avgRarity: 0, + statRarity: 1, + rarityScore: 0, + rarityScoreNormed: 0, + usedTraitsCount: item.attributes.length, + }; - item.attributes.forEach((a) => { - const attributeData = rarityObject[a.trait_type][a.value]; - // original buggy implementation (not ok if different nr. of attributes) - //attributeData.attributeRarityNormed = attributeData.attributeRarity * avgAttributesPerTrait / item.attributes.length; - item.rarity.avgRarity += attributeData.attributeFrequency; - item.rarity.statRarity *= attributeData.attributeFrequency; - item.rarity.rarityScore += attributeData.attributeRarity; - item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + // original buggy implementation (not ok if different nr. of attributes) + //attributeData.attributeRarityNormed = attributeData.attributeRarity * avgAttributesPerTrait / item.attributes.length; + item.rarity.avgRarity += attributeData.attributeFrequency; + item.rarity.statRarity *= attributeData.attributeFrequency; + item.rarity.rarityScore += attributeData.attributeRarity; + item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + }); + }); + } else if (network.metadataType == metadataTypes.rarities_JD) { + let z = []; + let avg = []; + + // calculate z(i,j) and avg(i) + for (let i = 0; i < metadataList.length; i++) { + for (let j = 0; j < metadataList.length; j++) { + if (i == j) continue; + + if (z[i] == null) { + z[i] = []; + } + + if (z[i][j] == null || z[j][i] == null) { + const commonTraitsCnt = getObjectCommonCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + const uniqueTraitsCnt = getObjectUniqueCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + + z[i][j] = commonTraitsCnt / uniqueTraitsCnt; + } + } + + // ps: length-1 because there's always an empty cell in matrix, where i == j + avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); + } + + // calculate z(i) + let jd = []; + let avgMax = Math.max(...avg); + let avgMin = Math.min(...avg); + + for (let i = 0; i < metadataList.length; i++) { + jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; + } + + const jd_asc = [...jd].sort(function (a, b) { + return a - b; }); + + // add JD rarity data to NFT/item + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarityScore = jd[i]; + metadataList[i].rarityRank = jd.length - jd_asc.indexOf(jd[i]); + } + + //console.log("z", z); + //console.log("avg", avg); + //console.log("avgMax", avgMax); + //console.log("avgMin", avgMin); + //console.log("jd", jd); + //console.log("ranks", ranks); + } +}; + +const getArrayCommonCnt = (arr1, arr2) => { + var cnt = 0; + for (var i = 0; i < arr1.length; ++i) { + for (var j = 0; j < arr2.length; ++j) { + if (arr1[i] == arr2[j]) { + cnt++; + } + } + } + return cnt; +}; +const getArrayUniqueCnt = (arr1, arr2) => { + return [...new Set(arr1.concat(arr2))].length; +}; + +const getObjectCommonCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + Object.entries(obj1).forEach((entry) => { + arr1.push(JSON.stringify(entry[1])); + }); + Object.entries(obj2).forEach((entry) => { + arr2.push(JSON.stringify(entry[1])); + }); + return getArrayCommonCnt(arr1, arr2); +}; +const getObjectUniqueCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + Object.entries(obj1).forEach((entry) => { + arr1.push(JSON.stringify(entry[1])); + }); + Object.entries(obj2).forEach((entry) => { + arr2.push(JSON.stringify(entry[1])); }); + return getArrayUniqueCnt(arr1, arr2); }; const addAttributes = (_element) => { @@ -494,7 +590,9 @@ const startCreating = async () => { hashlipsGiffer = new HashlipsGiffer( canvas, ctx, - `${buildDir}/${network.mediaDirPrefix ?? ""}${abstractedIndexes[0]}.gif`, + `${buildDir}/${network.mediaDirPrefix ?? ""}${ + abstractedIndexes[0] + }.gif`, gif.repeat, gif.quality, gif.delay @@ -546,7 +644,7 @@ const startCreating = async () => { } // build rarity (if needed) - if (network.metadataType == metadataTypes.rarities) { + if (network.metadataType != metadataTypes.basic) { addRarityMetadata(); } @@ -556,7 +654,7 @@ const startCreating = async () => { // save metadata.json if (network.metadataType == metadataTypes.basic) writeMetaData(JSON.stringify(metadataList, null, 2)); - else if (network.metadataType == metadataTypes.rarities) { + else { writeMetaData(JSON.stringify(rarityObject, null, 2)); } }; diff --git a/utils/common.js b/utils/common.js new file mode 100644 index 000000000..ed2446731 --- /dev/null +++ b/utils/common.js @@ -0,0 +1,41 @@ +const basePath = process.cwd(); +const fs = require("fs"); +const { network } = require(`${basePath}/src/config.js`); +const { metadataTypes } = require(`${basePath}/constants/network.js`); + +// get metadata for all generated NFTs/items +const getMetadataItems = () => { + if (network.metadataType == metadataTypes.basic) { + // get metadata from the _metadata.json file + return JSON.parse( + fs.readFileSync( + `${basePath}/build/${network.jsonDirPrefix ?? ""}${ + network.metadataFileName + }` + ) + ); + } else { + // get metadata from the individual metadata file + let rawData = []; + const jsonFilePattern = /^\d+.(json)$/i; + const files = fs.readdirSync( + `${basePath}/build/${network.jsonDirPrefix ?? ""}` + ); + files.forEach((file) => { + if (file.match(jsonFilePattern)) { + rawData.push( + JSON.parse( + fs.readFileSync( + `${basePath}/build/${network.jsonDirPrefix ?? ""}${file}` + ) + ) + ); + } + }); + return rawData; + } +}; + +module.exports = { + getMetadataItems, +}; diff --git a/utils/preview.js b/utils/preview.js index 824d623b9..52dce470c 100644 --- a/utils/preview.js +++ b/utils/preview.js @@ -3,13 +3,11 @@ const fs = require("fs"); const { createCanvas, loadImage } = require("canvas"); const buildDir = `${basePath}/build`; -const { preview } = require(`${basePath}/src/config.js`); +const { network, preview } = require(`${basePath}/src/config.js`); +const { getMetadataItems } = require(`${basePath}/utils/common.js`); // read json data -const rawdata = fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}` -); -const metadataList = JSON.parse(rawdata); +const metadataList = getMetadataItems(); const saveProjectPreviewImage = async (_data) => { // Extract from preview config @@ -35,7 +33,9 @@ const saveProjectPreviewImage = async (_data) => { for (let index = 0; index < _data.length; index++) { const nft = _data[index]; await loadImage( - `${buildDir}/${network.mediaDirPrefix ?? ""}$${nft.edition}.png` + `${buildDir}/${network.mediaDirPrefix ?? ""}${ + network.mediaFilePrefix ?? "" + }${nft.edition}.png` ).then((image) => { previewCtx.drawImage( image, diff --git a/utils/rarity.js b/utils/rarity.js index 78536dca3..f4efcbf4f 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -1,19 +1,23 @@ const basePath = process.cwd(); const fs = require("fs"); +const { exit } = require("process"); +const { metadataTypes } = require("../constants/network.js"); +const { network } = require("../src/config.js"); const layersDir = `${basePath}/layers`; const { layerConfigurations } = require(`${basePath}/src/config.js`); +const { getMetadataItems } = require(`${basePath}/utils/common.js`); const { getElements } = require("../src/main.js"); // read json data -let rawdata = fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`); -let data = JSON.parse(rawdata); +let data = getMetadataItems(); let editionSize = data.length; let rarityData = []; // intialize layers to chart +console.log("layerconfig", layerConfigurations); layerConfigurations.forEach((config) => { let layers = config.layersOrder; @@ -63,12 +67,15 @@ data.forEach((element) => { for (var layer in rarityData) { for (var attribute in rarityData[layer]) { // get chance - let chance = - ((rarityData[layer][attribute].occurrence / editionSize) * 100).toFixed(2); + let chance = ( + (rarityData[layer][attribute].occurrence / editionSize) * + 100 + ).toFixed(2); // show two decimal places in percent - rarityData[layer][attribute].occurrence = - `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; + rarityData[layer][ + attribute + ].occurrence = `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; } } diff --git a/utils/update_info.js b/utils/update_info.js index 72200ffd8..8d1e8a4d2 100644 --- a/utils/update_info.js +++ b/utils/update_info.js @@ -1,6 +1,8 @@ const basePath = process.cwd(); const { NETWORK } = require(`${basePath}/constants/network.js`); const fs = require("fs"); +const { metadataTypes } = require("../constants/network"); +const { getMetadataItems } = require(`${basePath}/utils/common.js`); const { baseUri, @@ -11,22 +13,23 @@ const { } = require(`${basePath}/src/config.js`); // read json data -let rawdata = fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}` -); -let data = JSON.parse(rawdata); +let data = getMetadataItems(); +let idx = network.startIdx; data.forEach((item) => { // general metadata item.description = description; + if (network.metadataType != metadataTypes.basic) { + item.name = `${namePrefix} #${idx++}`; + } else { + item.name = `${namePrefix} #${item.edition}`; + } // custom metadata if (network == NETWORK.eth) { item.image = `${baseUri}/${item.edition}.png`; - item.name = `${namePrefix} #${item.edition}`; } if (network == NETWORK.sol) { - item.name = `${namePrefix} #${item.edition}`; item.creators = solanaMetadata.creators; } @@ -36,10 +39,14 @@ data.forEach((item) => { ); }); -fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`, - JSON.stringify(data, null, 2) -); +if (network.metadataType == metadataTypes.basic) { + fs.writeFileSync( + `${basePath}/build/${network.jsonDirPrefix ?? ""}${ + network.metadataFileName + }`, + JSON.stringify(data, null, 2) + ); +} if (network == NETWORK.sol) { console.log(`Updated description for images to ===> ${description}`); From 3f2c93276226f1fa72d80612089739505cc7433a Mon Sep 17 00:00:00 2001 From: johnykes Date: Mon, 18 Apr 2022 17:46:51 +0300 Subject: [PATCH 07/19] existing scripts adapted for the new customisations --- constants/network.js | 4 +- src/main.js | 106 +++++++++++-------------------------------- utils/common.js | 49 +++++++++++++++++++- 3 files changed, 76 insertions(+), 83 deletions(-) diff --git a/constants/network.js b/constants/network.js index cb552aa1e..a8d31d4cf 100644 --- a/constants/network.js +++ b/constants/network.js @@ -5,9 +5,9 @@ const metadataTypes = { /// metadata file will contain only rarity data for traits & attributes /// individual metadata file will also contain rarity data - // rarities calculated using Trait Rarity (less accurate) + // rarities calculated using Trait Rarity (not accurate/recommended) rarities_TR: 1, - // rarities calculated using JaccardDistances (most accurate) + // rarities calculated using JaccardDistances (accurate/recommended) rarities_JD: 2, }; diff --git a/src/main.js b/src/main.js index 0c5dc1a72..7765f1229 100644 --- a/src/main.js +++ b/src/main.js @@ -22,6 +22,10 @@ const { solanaMetadata, gif, } = require(`${basePath}/src/config.js`); +const { + getObjectCommonCnt, + getObjectUniqueCnt, +} = require(`${basePath}/utils/common.js`); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); ctx.imageSmoothingEnabled = format.smoothing; @@ -190,10 +194,6 @@ const addMetadata = (_dna, _edition) => { }; const addRarityMetadata = () => { - // currently using https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts logic - // ps: the only difference is that the attributeRarityNormed is calculated only once (in case there are NFTs with less/more attributes than others) - // todo: add multiple rarity implementations - let traitOccurances = []; let totalAttributesCnt = 0; @@ -217,10 +217,7 @@ const addRarityMetadata = () => { }); }); - const totalLayersCnt = Object.keys(traitOccurances).length; - const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; - - // calculate rarity for all traits/layers & attributes/assets + // general rarity metadata for all traits/layers & attributes/assets Object.entries(rarityObject).forEach((entry) => { const layer = entry[0]; const assets = entry[1]; @@ -230,16 +227,25 @@ const addRarityMetadata = () => { asset[1].traitOccurance = traitOccurances[layer]; asset[1].traitOccurancePercentage = (metadataList.length / asset[1].traitOccurance) * 100; - asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; // attribute/asset related - asset[1].attributeFrequency = - asset[1].attributeOccurrence / metadataList.length; - asset[1].attributeRarity = - metadataList.length / asset[1].attributeOccurrence; - // ugly fix for the original implementation - asset[1].attributeRarityNormed = - asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); + asset[1].attributeOccurrencePercentage = + (asset[1].attributeOccurrence / metadataList.length) * 100; + + // rarity algorithm specific metadata + if (network.metadataType == metadataTypes.rarities_TR) { + // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts + // ps: the only difference being that attributeRarityNormed is calculated only once + const totalLayersCnt = Object.keys(traitOccurances).length; + const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; + asset[1].attributeFrequency = + asset[1].attributeOccurrence / metadataList.length; + asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; + asset[1].attributeRarity = + metadataList.length / asset[1].attributeOccurrence; + asset[1].attributeRarityNormed = + asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); + } }); }); @@ -256,8 +262,6 @@ const addRarityMetadata = () => { item.attributes.forEach((a) => { const attributeData = rarityObject[a.trait_type][a.value]; - // original buggy implementation (not ok if different nr. of attributes) - //attributeData.attributeRarityNormed = attributeData.attributeRarity * avgAttributesPerTrait / item.attributes.length; item.rarity.avgRarity += attributeData.attributeFrequency; item.rarity.statRarity *= attributeData.attributeFrequency; item.rarity.rarityScore += attributeData.attributeRarity; @@ -310,55 +314,12 @@ const addRarityMetadata = () => { // add JD rarity data to NFT/item for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarityScore = jd[i]; - metadataList[i].rarityRank = jd.length - jd_asc.indexOf(jd[i]); - } - - //console.log("z", z); - //console.log("avg", avg); - //console.log("avgMax", avgMax); - //console.log("avgMin", avgMin); - //console.log("jd", jd); - //console.log("ranks", ranks); - } -}; - -const getArrayCommonCnt = (arr1, arr2) => { - var cnt = 0; - for (var i = 0; i < arr1.length; ++i) { - for (var j = 0; j < arr2.length; ++j) { - if (arr1[i] == arr2[j]) { - cnt++; - } + metadataList[i].rarity = { + score: jd[i], + rank: jd.length - jd_asc.indexOf(jd[i]), + }; } } - return cnt; -}; -const getArrayUniqueCnt = (arr1, arr2) => { - return [...new Set(arr1.concat(arr2))].length; -}; - -const getObjectCommonCnt = (obj1, obj2) => { - let arr1 = []; - let arr2 = []; - Object.entries(obj1).forEach((entry) => { - arr1.push(JSON.stringify(entry[1])); - }); - Object.entries(obj2).forEach((entry) => { - arr2.push(JSON.stringify(entry[1])); - }); - return getArrayCommonCnt(arr1, arr2); -}; -const getObjectUniqueCnt = (obj1, obj2) => { - let arr1 = []; - let arr2 = []; - Object.entries(obj1).forEach((entry) => { - arr1.push(JSON.stringify(entry[1])); - }); - Object.entries(obj2).forEach((entry) => { - arr2.push(JSON.stringify(entry[1])); - }); - return getArrayUniqueCnt(arr1, arr2); }; const addAttributes = (_element) => { @@ -500,19 +461,6 @@ const writeMetaData = (_data) => { ); }; -/*const saveMetaDataSingleFile = (_editionCount) => { - let metadata = metadataList.find((meta) => meta.edition == _editionCount); - debugLogs - ? console.log( - `Writing metadata for ${_editionCount}: ${JSON.stringify(metadata)}` - ) - : null; - fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix}${_editionCount}.json`, - JSON.stringify(metadata, null, 2) - ); -};*/ - const saveIndividualMetadataFiles = (abstractedIndexes) => { let idx = 0; metadataList.forEach((item) => { @@ -643,7 +591,7 @@ const startCreating = async () => { layerConfigIndex++; } - // build rarity (if needed) + // build rarity if (network.metadataType != metadataTypes.basic) { addRarityMetadata(); } diff --git a/utils/common.js b/utils/common.js index ed2446731..a469a198a 100644 --- a/utils/common.js +++ b/utils/common.js @@ -3,7 +3,7 @@ const fs = require("fs"); const { network } = require(`${basePath}/src/config.js`); const { metadataTypes } = require(`${basePath}/constants/network.js`); -// get metadata for all generated NFTs/items +// get metadata of all generated NFTs/items const getMetadataItems = () => { if (network.metadataType == metadataTypes.basic) { // get metadata from the _metadata.json file @@ -15,7 +15,7 @@ const getMetadataItems = () => { ) ); } else { - // get metadata from the individual metadata file + // get metadata from the individual metadata files let rawData = []; const jsonFilePattern = /^\d+.(json)$/i; const files = fs.readdirSync( @@ -36,6 +36,51 @@ const getMetadataItems = () => { } }; +// get common elements counter of 2 arrays +const getArrayCommonCnt = (arr1, arr2) => { + var cnt = 0; + for (var i = 0; i < arr1.length; ++i) { + for (var j = 0; j < arr2.length; ++j) { + if (arr1[i] == arr2[j]) { + cnt++; + } + } + } + return cnt; +}; +// get unique elements counter of 2 arrays +const getArrayUniqueCnt = (arr1, arr2) => { + return [...new Set(arr1.concat(arr2))].length; +}; +// get common elements counter of 2 objects +const getObjectCommonCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + Object.entries(obj1).forEach((entry) => { + arr1.push(JSON.stringify(entry[1])); + }); + Object.entries(obj2).forEach((entry) => { + arr2.push(JSON.stringify(entry[1])); + }); + return getArrayCommonCnt(arr1, arr2); +}; +// get unique elements counter of 2 objects +const getObjectUniqueCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + Object.entries(obj1).forEach((entry) => { + arr1.push(JSON.stringify(entry[1])); + }); + Object.entries(obj2).forEach((entry) => { + arr2.push(JSON.stringify(entry[1])); + }); + return getArrayUniqueCnt(arr1, arr2); +}; + module.exports = { getMetadataItems, + getArrayCommonCnt, + getArrayUniqueCnt, + getObjectCommonCnt, + getObjectUniqueCnt, }; From 1b09f78b112db674fb51d64d98767287b24a7929 Mon Sep 17 00:00:00 2001 From: johnykes Date: Mon, 18 Apr 2022 18:39:45 +0300 Subject: [PATCH 08/19] clean a little bit --- constants/network.js | 8 +-- src/main.js | 125 +++++++++++++++++++++++-------------------- utils/pixelate.js | 2 +- 3 files changed, 72 insertions(+), 63 deletions(-) diff --git a/constants/network.js b/constants/network.js index a8d31d4cf..cdc28e19c 100644 --- a/constants/network.js +++ b/constants/network.js @@ -5,10 +5,10 @@ const metadataTypes = { /// metadata file will contain only rarity data for traits & attributes /// individual metadata file will also contain rarity data - // rarities calculated using Trait Rarity (not accurate/recommended) - rarities_TR: 1, - // rarities calculated using JaccardDistances (accurate/recommended) - rarities_JD: 2, + // rarities calculated using JaccardDistances (most accurate/recommended) + rarities_JD: 1, + // rarities calculated using Trait Rarity (not recommended) + rarities_TR: 2, }; const NETWORK = { diff --git a/src/main.js b/src/main.js index 7765f1229..bdc39e1d2 100644 --- a/src/main.js +++ b/src/main.js @@ -250,75 +250,84 @@ const addRarityMetadata = () => { }); // calculate rarity for each item/NFT - if (network.metadataType == metadataTypes.rarities_TR) { - metadataList.forEach((item) => { - item.rarity = { - avgRarity: 0, - statRarity: 1, - rarityScore: 0, - rarityScoreNormed: 0, - usedTraitsCount: item.attributes.length, - }; - - item.attributes.forEach((a) => { - const attributeData = rarityObject[a.trait_type][a.value]; - item.rarity.avgRarity += attributeData.attributeFrequency; - item.rarity.statRarity *= attributeData.attributeFrequency; - item.rarity.rarityScore += attributeData.attributeRarity; - item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; - }); - }); - } else if (network.metadataType == metadataTypes.rarities_JD) { - let z = []; - let avg = []; + switch (network.metadataType) { + case metadataTypes.rarities_JD: { + let z = []; + let avg = []; + + // calculate z(i,j) and avg(i) + for (let i = 0; i < metadataList.length; i++) { + for (let j = 0; j < metadataList.length; j++) { + if (i == j) continue; + + if (z[i] == null) { + z[i] = []; + } - // calculate z(i,j) and avg(i) - for (let i = 0; i < metadataList.length; i++) { - for (let j = 0; j < metadataList.length; j++) { - if (i == j) continue; + if (z[i][j] == null || z[j][i] == null) { + const commonTraitsCnt = getObjectCommonCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + const uniqueTraitsCnt = getObjectUniqueCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); - if (z[i] == null) { - z[i] = []; + z[i][j] = commonTraitsCnt / uniqueTraitsCnt; + } } - if (z[i][j] == null || z[j][i] == null) { - const commonTraitsCnt = getObjectCommonCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - const uniqueTraitsCnt = getObjectUniqueCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - - z[i][j] = commonTraitsCnt / uniqueTraitsCnt; - } + // ps: length-1 because there's always an empty cell in matrix, where i == j + avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); } - // ps: length-1 because there's always an empty cell in matrix, where i == j - avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); - } + // calculate z(i) + let jd = []; + let avgMax = Math.max(...avg); + let avgMin = Math.min(...avg); - // calculate z(i) - let jd = []; - let avgMax = Math.max(...avg); - let avgMin = Math.min(...avg); + for (let i = 0; i < metadataList.length; i++) { + jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; + } - for (let i = 0; i < metadataList.length; i++) { - jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; - } + const jd_asc = [...jd].sort(function (a, b) { + return a - b; + }); - const jd_asc = [...jd].sort(function (a, b) { - return a - b; - }); + // add JD rarity data to NFT/item + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity = { + score: jd[i], + rank: jd.length - jd_asc.indexOf(jd[i]), + }; + } + break; + } - // add JD rarity data to NFT/item - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity = { - score: jd[i], - rank: jd.length - jd_asc.indexOf(jd[i]), - }; + case metadataTypes.rarities_TR: { + metadataList.forEach((item) => { + item.rarity = { + avgRarity: 0, + statRarity: 1, + rarityScore: 0, + rarityScoreNormed: 0, + usedTraitsCount: item.attributes.length, + }; + + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + item.rarity.avgRarity += attributeData.attributeFrequency; + item.rarity.statRarity *= attributeData.attributeFrequency; + item.rarity.rarityScore += attributeData.attributeRarity; + item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + }); + }); + break; } + + default: + break; } }; diff --git a/utils/pixelate.js b/utils/pixelate.js index cc3ee24cf..50a099b6e 100644 --- a/utils/pixelate.js +++ b/utils/pixelate.js @@ -2,9 +2,9 @@ const fs = require("fs"); const path = require("path"); const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); +const { format, network, pixelFormat } = require(`${basePath}/src/config.js`); const buildDir = `${basePath}/build/pixel_images`; const inputDir = `${basePath}/build/${network.mediaDirPrefix ?? ""}`; -const { format, pixelFormat } = require(`${basePath}/src/config.js`); const console = require("console"); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); From 8ae250a0405067012bdd4c8ad7935272659161fb Mon Sep 17 00:00:00 2001 From: johnykes Date: Mon, 18 Apr 2022 18:55:03 +0300 Subject: [PATCH 09/19] rename TR algorithm to `Common` --- constants/network.js | 4 ++-- src/main.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/constants/network.js b/constants/network.js index cdc28e19c..f34b7284a 100644 --- a/constants/network.js +++ b/constants/network.js @@ -7,8 +7,8 @@ const metadataTypes = { /// individual metadata file will also contain rarity data // rarities calculated using JaccardDistances (most accurate/recommended) rarities_JD: 1, - // rarities calculated using Trait Rarity (not recommended) - rarities_TR: 2, + // rarities calculated using Trait Rarity & Statistical Rarity (not recommended) + rarities_Common: 2, }; const NETWORK = { diff --git a/src/main.js b/src/main.js index bdc39e1d2..c3f6be0fb 100644 --- a/src/main.js +++ b/src/main.js @@ -233,7 +233,7 @@ const addRarityMetadata = () => { (asset[1].attributeOccurrence / metadataList.length) * 100; // rarity algorithm specific metadata - if (network.metadataType == metadataTypes.rarities_TR) { + if (network.metadataType == metadataTypes.rarities_Common) { // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts // ps: the only difference being that attributeRarityNormed is calculated only once const totalLayersCnt = Object.keys(traitOccurances).length; @@ -305,7 +305,7 @@ const addRarityMetadata = () => { break; } - case metadataTypes.rarities_TR: { + case metadataTypes.rarities_Common: { metadataList.forEach((item) => { item.rarity = { avgRarity: 0, From a60fcf2264dd9f3c9272c83c926277af4ad1a11f Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 19 Apr 2022 13:39:55 +0300 Subject: [PATCH 10/19] split TR & SR rarity algorithms; add rank for all rarity algorithms --- constants/network.js | 23 ++++-- src/main.js | 180 +++++++++++++++++++++++++++---------------- 2 files changed, 127 insertions(+), 76 deletions(-) diff --git a/constants/network.js b/constants/network.js index f34b7284a..c4b9ffa17 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,14 +1,18 @@ const metadataTypes = { // metadata file will contain all individual metadata files (common for eth$, sol$) - // no rarities + // no rarities at all basic: 0, + // metadata file will contain only rarity data for traits & attributes (common for egld$) + // if rarityAlgorithm provided, individual metadata file will also contain rarity data + rarities: 1, +}; - /// metadata file will contain only rarity data for traits & attributes - /// individual metadata file will also contain rarity data - // rarities calculated using JaccardDistances (most accurate/recommended) - rarities_JD: 1, - // rarities calculated using Trait Rarity & Statistical Rarity (not recommended) - rarities_Common: 2, +const rarityAlgorithms = { + none: 0, + JaccardDistances: 1, // most accurate / recommended + TraitRarity: 2, + StatisticalRarity: 3, + TraitAndStatisticalRarity: 4, // TraitRarity & StatisticalRarity combined }; const NETWORK = { @@ -16,7 +20,9 @@ const NETWORK = { name: "egld", startIdx: 1, metadataFileName: "_metadata.json", - metadataType: metadataTypes.rarities_JD, + metadataType: metadataTypes.rarities, + rarityAlgorithm: rarityAlgorithms.JaccardDistances, + includeRank: true }, eth: { name: "eth", @@ -41,4 +47,5 @@ const NETWORK = { module.exports = { NETWORK, metadataTypes, + rarityAlgorithms, }; diff --git a/src/main.js b/src/main.js index c3f6be0fb..18bc8ae9b 100644 --- a/src/main.js +++ b/src/main.js @@ -1,5 +1,9 @@ const basePath = process.cwd(); -const { NETWORK, metadataTypes } = require(`${basePath}/constants/network.js`); +const { + NETWORK, + metadataTypes, + rarityAlgorithms, +} = require(`${basePath}/constants/network.js`); const fs = require("fs"); const sha1 = require(`${basePath}/node_modules/sha1`); const { createCanvas, loadImage } = require(`${basePath}/node_modules/canvas`); @@ -232,8 +236,12 @@ const addRarityMetadata = () => { asset[1].attributeOccurrencePercentage = (asset[1].attributeOccurrence / metadataList.length) * 100; - // rarity algorithm specific metadata - if (network.metadataType == metadataTypes.rarities_Common) { + // TR / SR algorithms specific metadata + if ( + network.rarityAlgorithm == rarityAlgorithms.TraitRarity || + network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts // ps: the only difference being that attributeRarityNormed is calculated only once const totalLayersCnt = Object.keys(traitOccurances).length; @@ -250,84 +258,120 @@ const addRarityMetadata = () => { }); // calculate rarity for each item/NFT - switch (network.metadataType) { - case metadataTypes.rarities_JD: { - let z = []; - let avg = []; - - // calculate z(i,j) and avg(i) - for (let i = 0; i < metadataList.length; i++) { - for (let j = 0; j < metadataList.length; j++) { - if (i == j) continue; - - if (z[i] == null) { - z[i] = []; - } + if (network.rarityAlgorithm == rarityAlgorithms.none) { + return; + } else if (network.rarityAlgorithm == rarityAlgorithms.JaccardDistances) { + let z = []; + let avg = []; + + // calculate z(i,j) and avg(i) + for (let i = 0; i < metadataList.length; i++) { + for (let j = 0; j < metadataList.length; j++) { + if (i == j) continue; + + if (z[i] == null) { + z[i] = []; + } - if (z[i][j] == null || z[j][i] == null) { - const commonTraitsCnt = getObjectCommonCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - const uniqueTraitsCnt = getObjectUniqueCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); + if (z[i][j] == null || z[j][i] == null) { + const commonTraitsCnt = getObjectCommonCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + const uniqueTraitsCnt = getObjectUniqueCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); - z[i][j] = commonTraitsCnt / uniqueTraitsCnt; - } + z[i][j] = commonTraitsCnt / uniqueTraitsCnt; } - - // ps: length-1 because there's always an empty cell in matrix, where i == j - avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); } - // calculate z(i) - let jd = []; - let avgMax = Math.max(...avg); - let avgMin = Math.min(...avg); + // ps: length-1 because there's always an empty cell in matrix, where i == j + avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); + } - for (let i = 0; i < metadataList.length; i++) { - jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; - } + // calculate z(i) + let jd = []; + let avgMax = Math.max(...avg); + let avgMin = Math.min(...avg); - const jd_asc = [...jd].sort(function (a, b) { - return a - b; - }); + for (let i = 0; i < metadataList.length; i++) { + jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; + } - // add JD rarity data to NFT/item - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity = { - score: jd[i], - rank: jd.length - jd_asc.indexOf(jd[i]), - }; + const jd_asc = [...jd].sort(function (a, b) { + return a - b; + }); + + // add JD rarity data to NFT/item + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity = { + score: jd[i], + }; + if (network.includeRank) { + metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); } - break; } + } else if ( + network.rarityAlgorithm == rarityAlgorithms.TraitRarity || + network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { + metadataList.forEach((item) => { + item.rarity = { + usedTraitsCount: item.attributes.length, + }; + // TR specific + if ( + network.rarityAlgorithm == rarityAlgorithms.TraitRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { + item.rarity.avgRarity = 0; + item.rarity.rarityScore = 0; + item.rarity.rarityScoreNormed = 0; + } + // SR specific + if ( + network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { + item.rarity.statRarity = 1; + } - case metadataTypes.rarities_Common: { - metadataList.forEach((item) => { - item.rarity = { - avgRarity: 0, - statRarity: 1, - rarityScore: 0, - rarityScoreNormed: 0, - usedTraitsCount: item.attributes.length, - }; - - item.attributes.forEach((a) => { - const attributeData = rarityObject[a.trait_type][a.value]; + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + if ( + network.rarityAlgorithm == rarityAlgorithms.TraitRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { item.rarity.avgRarity += attributeData.attributeFrequency; - item.rarity.statRarity *= attributeData.attributeFrequency; item.rarity.rarityScore += attributeData.attributeRarity; item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; - }); + } + // SR specific + if ( + network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || + network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity + ) { + item.rarity.statRarity *= attributeData.attributeFrequency; + } }); - break; - } + }); - default: - break; + // add rarity rank + if (network.includeRank) { + const metadataList_asc = [...metadataList].sort(function (a, b) { + return ( + a.rarity.rarityScore - b.rarity.rarityScore ?? + b.rarity.statRarity - a.rarity.statRarity + ); + }); + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity.rank = + metadataList.length - metadataList_asc.indexOf(metadataList[i]); + } + } } }; @@ -601,7 +645,7 @@ const startCreating = async () => { } // build rarity - if (network.metadataType != metadataTypes.basic) { + if (network.metadataType == metadataTypes.rarities) { addRarityMetadata(); } @@ -609,9 +653,9 @@ const startCreating = async () => { saveIndividualMetadataFiles(abstractedIndexesBackup); // save metadata.json - if (network.metadataType == metadataTypes.basic) + if (network.metadataType == metadataTypes.basic) { writeMetaData(JSON.stringify(metadataList, null, 2)); - else { + } else { writeMetaData(JSON.stringify(rarityObject, null, 2)); } }; From 68084fd1badfb8ab31fea4138b4f761223446338 Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 26 Apr 2022 00:25:35 +0300 Subject: [PATCH 11/19] split constants files; split rarity & metadata into different components; add defaults values for network.js options --- README.md | 48 +++- constants/metadata.js | 10 + constants/network.js | 51 ++-- constants/rarity.js | 9 + package.json | 12 +- src/config.js | 4 +- src/main.js | 292 +++-------------------- utils/common.js | 32 +-- utils/metadata.js | 73 ++++++ utils/rarity.js | 283 +++++++++++++++------- utils/{ => scripts}/generate_metadata.js | 2 +- utils/{ => scripts}/pixelate.js | 2 +- utils/{ => scripts}/preview.js | 4 +- utils/{ => scripts}/preview_gif.js | 2 +- utils/scripts/rarity.js | 89 +++++++ utils/{ => scripts}/update_info.js | 16 +- 16 files changed, 512 insertions(+), 417 deletions(-) create mode 100644 constants/metadata.js create mode 100644 constants/rarity.js create mode 100644 utils/metadata.js rename utils/{ => scripts}/generate_metadata.js (98%) rename utils/{ => scripts}/pixelate.js (97%) rename utils/{ => scripts}/preview.js (95%) rename utils/{ => scripts}/preview_gif.js (97%) create mode 100644 utils/scripts/rarity.js rename utils/{ => scripts}/update_info.js (77%) diff --git a/README.md b/README.md index 2ac0970b7..df9bb7cbb 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ If you want to have logs to debug and see what is happening when you generate im If you want to play around with different blending modes, you can add a `blend: MODE.colorBurn` field to the layersOrder `options` object. -If you need a layers to have a different opacity then you can add the `opacity: 0.7` field to the layersOrder `options` object as well. +If you need a layer to have a different opacity then you can add the `opacity: 0.7` field to the layersOrder `options` object as well. If you want to have a layer _ignored_ in the DNA uniqueness check, you can set `bypassDNA: true` in the `options` object. This has the effect of making sure the rest of the traits are unique while not considering the `Background` Layers as traits, for example. The layers _are_ included in the final image. @@ -178,7 +178,7 @@ const MODE = { }; ``` -When you are ready, run the following command and your outputted art will be in the `build${network.mediaDirPrefix}` directory and the json in the `build${network.jsonDirPrefix}` directory: +When you are ready, run the following command and your outputted art will be in the `build/images` directory and the json in the `build/json` directory: ```sh npm run build @@ -190,7 +190,7 @@ or node index.js ``` -The program will output all the images in the `build${network.mediaDirPrefix}` directory along with the metadata files in the `build${network.jsonDirPrefix}` directory. Each collection will have a `${network.metadataFileName}` file that consists of all the metadata in the collection inside the `build${network.jsonDirPrefix}` directory. The `build${network.jsonDirPrefix}` folder also will contain all the single json files that represent each image file. The single json file of a image will look something like this: +The program will output all the images in the `build/media` directory along with the metadata files in the `build/json` directory. Each collection will have a `_metadata.json` file that consists of all the metadata in the collection inside the `build/json` directory. The `build/json` folder also will contain all the single json files that represent each image file. The single json file of a image will look something like this: ```json { @@ -229,6 +229,48 @@ const extraMetadata = {}; That's it, you're done. +## Output customization +Depending on the minting process / marketplace you choose, you will need to respect an output folder structure or you may want to include rarity metadata.\ +In order to get your desired output structure / rarity metadata, you can choose from the already created networks/standards that can be found in `network.js` and update the network value in `config.js` +``` +// config.js +const network = NETWORK.egld; +``` +or you can create your own network/standard in `network.js`. +``` +// network.js +const NETWORK = { + egld: { + name: "egld", + startIdx: 1, + metadataFileName: "_metadata.json", + metadataType: METADATA.rarities, + rarityAlgorithm: RARITY.JaccardDistances, + includeRank: true + }, + ... +} +``` +The `metadataType` and `rarityAlgorithm` options can be also found in `network.js` +``` +const METADATA = { + // metadata file will contain all individual metadata files (common for eth$, sol$) + // no rarities at all + basic: 0, + // metadata file will contain only rarity data for traits & attributes (common for egld$) + // if rarityAlgorithm provided, individual metadata files will also contain rarity data + rarities: 1, +}; + +const RARITY = { + none: 0, + JaccardDistances: 1, // most accurate / recommended + TraitRarity: 2, + StatisticalRarity: 3, + TraitAndStatisticalRarity: 4, +}; +``` + ## Utils ### Updating baseUri for IPFS and description diff --git a/constants/metadata.js b/constants/metadata.js new file mode 100644 index 000000000..13059fa8d --- /dev/null +++ b/constants/metadata.js @@ -0,0 +1,10 @@ +const METADATA = { + // metadata file will contain all individual metadata files (common for eth$, sol$) + // no rarities at all + basic: 0, + // metadata file will contain only rarity data for traits & attributes (common for egld$) + // if rarityAlgorithm provided, individual metadata files will also contain rarity data + rarities: 1, +}; + +module.exports = { METADATA }; diff --git a/constants/network.js b/constants/network.js index c4b9ffa17..46aff9aa0 100644 --- a/constants/network.js +++ b/constants/network.js @@ -1,51 +1,44 @@ -const metadataTypes = { - // metadata file will contain all individual metadata files (common for eth$, sol$) - // no rarities at all - basic: 0, - // metadata file will contain only rarity data for traits & attributes (common for egld$) - // if rarityAlgorithm provided, individual metadata file will also contain rarity data - rarities: 1, -}; +const { METADATA } = require("../constants/metadata"); +const { RARITY } = require("../constants/rarity"); -const rarityAlgorithms = { - none: 0, - JaccardDistances: 1, // most accurate / recommended - TraitRarity: 2, - StatisticalRarity: 3, - TraitAndStatisticalRarity: 4, // TraitRarity & StatisticalRarity combined +const defaults = { + startIdx: 1, + jsonDirPrefix: "", + mediaDirPrefix: "", + mediaFilePrefix: "", + metadataFileName: "_metadata.json", + metadataType: METADATA.basic, + rarityAlgorithm: RARITY.none, + includeRank: false, }; const NETWORK = { - egld: { - name: "egld", - startIdx: 1, - metadataFileName: "_metadata.json", - metadataType: metadataTypes.rarities, - rarityAlgorithm: rarityAlgorithms.JaccardDistances, - includeRank: true - }, eth: { + ...defaults, name: "eth", - startIdx: 1, jsonDirPrefix: "json/", mediaDirPrefix: "media/", mediaFilePrefix: "$", - metadataFileName: "_metadata.json", - metadataType: metadataTypes.basic, }, sol: { + ...defaults, name: "sol", startIdx: 0, jsonDirPrefix: "json/", mediaDirPrefix: "media/", mediaFilePrefix: "$", - metadataFileName: "_metadata.json", - metadataType: metadataTypes.basic, + }, + egld: { + ...defaults, + name: "egld", + metadataType: METADATA.rarities, + rarityAlgorithm: RARITY.JaccardDistances, + includeRank: true, }, }; module.exports = { NETWORK, - metadataTypes, - rarityAlgorithms, + METADATA, + RARITY, }; diff --git a/constants/rarity.js b/constants/rarity.js new file mode 100644 index 000000000..a1b4697d8 --- /dev/null +++ b/constants/rarity.js @@ -0,0 +1,9 @@ +const RARITY = { + none: 0, + JaccardDistances: 1, // most accurate / recommended + TraitRarity: 2, + StatisticalRarity: 3, + TraitAndStatisticalRarity: 4, +}; + +module.exports = { RARITY }; diff --git a/package.json b/package.json index 141abf44e..203793880 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,12 @@ "scripts": { "build": "node index.js", "generate": "node index.js", - "rarity": "node utils/rarity.js", - "preview": "node utils/preview.js", - "pixelate": "node utils/pixelate.js", - "update_info": "node utils/update_info.js", - "preview_gif": "node utils/preview_gif.js", - "generate_metadata": "node utils/generate_metadata.js" + "rarity": "node utils/scripts/rarity.js", + "preview": "node utils/scripts/preview.js", + "pixelate": "node utils/scripts/pixelate.js", + "update_info": "node utils/scripts/update_info.js", + "preview_gif": "node utils/scripts/preview_gif.js", + "generate_metadata": "node utils/scripts/generate_metadata.js" }, "author": "Daniel Eugene Botha (HashLips)", "license": "MIT", diff --git a/src/config.js b/src/config.js index 576528ffd..adb365d4a 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.egld; +const network = NETWORK.eth; // General metadata const namePrefix = "Your Collection"; @@ -37,7 +37,7 @@ const layerConfigurations = [ { name: "Bottom lid" }, { name: "Top lid" }, ], - }, + } ]; const shuffleLayerConfigurations = false; diff --git a/src/main.js b/src/main.js index 18bc8ae9b..a98dbcad0 100644 --- a/src/main.js +++ b/src/main.js @@ -1,9 +1,7 @@ const basePath = process.cwd(); -const { - NETWORK, - metadataTypes, - rarityAlgorithms, -} = require(`${basePath}/constants/network.js`); +const { NETWORK } = require(`${basePath}/constants/network.js`); +const { METADATA } = require(`${basePath}/constants/metadata.js`); +const { RARITY } = require(`${basePath}/constants/rarity.js`); const fs = require("fs"); const sha1 = require(`${basePath}/node_modules/sha1`); const { createCanvas, loadImage } = require(`${basePath}/node_modules/canvas`); @@ -26,6 +24,11 @@ const { solanaMetadata, gif, } = require(`${basePath}/src/config.js`); +const { createMetadataItem } = require(`${basePath}/utils/metadata.js`); +const { + getGeneralRarity, + getItemsRarity, +} = require(`${basePath}/utils/rarity.js`); const { getObjectCommonCnt, getObjectUniqueCnt, @@ -36,7 +39,6 @@ ctx.imageSmoothingEnabled = format.smoothing; var metadataList = []; var attributesList = []; var dnaList = new Set(); -let rarityObject = {}; const DNA_DELIMITER = "-"; const HashlipsGiffer = require(`${basePath}/modules/HashlipsGiffer.js`); @@ -49,9 +51,9 @@ const buildSetup = () => { fs.mkdirSync(buildDir); if (network.jsonDirPrefix) - fs.mkdirSync(`${buildDir}/${network.jsonDirPrefix ?? ""}`); + fs.mkdirSync(`${buildDir}/${network.jsonDirPrefix}`); if (network.mediaDirPrefix) - fs.mkdirSync(`${buildDir}/${network.mediaDirPrefix ?? ""}`); + fs.mkdirSync(`${buildDir}/${network.mediaDirPrefix}`); }; const getRarityWeight = (_str) => { @@ -121,9 +123,7 @@ const layersSetup = (layersOrder) => { const saveImage = (_editionCount) => { fs.writeFileSync( - `${buildDir}/${network.mediaDirPrefix ?? ""}${ - network.mediaFilePrefix ?? "" - }${_editionCount}.png`, + `${buildDir}/${network.mediaDirPrefix}${network.mediaFilePrefix}${_editionCount}.png`, canvas.toBuffer("image/png") ); }; @@ -139,242 +139,6 @@ const drawBackground = () => { ctx.fillRect(0, 0, format.width, format.height); }; -const addMetadata = (_dna, _edition) => { - let tempMetadata = {}; - - switch (network) { - case NETWORK.egld: { - tempMetadata = { - description: description, - dna: sha1(_dna), - ...extraMetadata, - attributes: attributesList, - rarities: {}, - compiler: "HashLips Art Engine", - }; - } - case NETWORK.eth: { - tempMetadata = { - name: `${namePrefix} #${_edition}`, - description: description, - image: `${baseUri}/${_edition}.png`, - dna: sha1(_dna), - edition: _edition, - date: Date.now(), - attributes: attributesList, - ...extraMetadata, - compiler: "HashLips Art Engine", - }; - break; - } - case NETWORK.sol: { - tempMetadata = { - name: `${namePrefix} #${_edition}`, - symbol: solanaMetadata.symbol, - description: description, - seller_fee_basis_points: solanaMetadata.seller_fee_basis_points, - image: `${_edition}.png`, - external_url: solanaMetadata.external_url, - edition: _edition, - ...extraMetadata, - attributes: attributesList, - properties: { - files: [ - { - uri: `${_edition}.png`, - type: "image/png", - }, - ], - category: "image", - creators: solanaMetadata.creators, - }, - }; - break; - } - } - - metadataList.push(tempMetadata); - attributesList = []; -}; - -const addRarityMetadata = () => { - let traitOccurances = []; - let totalAttributesCnt = 0; - - // count occurrences for all traits/layers & attributes/assets - metadataList.forEach((item) => { - item.attributes.forEach((a) => { - if (rarityObject[a.trait_type] != null) { - if (rarityObject[a.trait_type][a.value] != null) { - rarityObject[a.trait_type][a.value].attributeOccurrence++; - } else { - rarityObject[a.trait_type][a.value] = { attributeOccurrence: 1 }; - totalAttributesCnt++; - } - - traitOccurances[a.trait_type]++; - } else { - rarityObject[a.trait_type] = { [a.value]: { attributeOccurrence: 1 } }; - traitOccurances[a.trait_type] = 1; - totalAttributesCnt++; - } - }); - }); - - // general rarity metadata for all traits/layers & attributes/assets - Object.entries(rarityObject).forEach((entry) => { - const layer = entry[0]; - const assets = entry[1]; - - Object.entries(assets).forEach((asset) => { - // trait/layer related - asset[1].traitOccurance = traitOccurances[layer]; - asset[1].traitOccurancePercentage = - (metadataList.length / asset[1].traitOccurance) * 100; - - // attribute/asset related - asset[1].attributeOccurrencePercentage = - (asset[1].attributeOccurrence / metadataList.length) * 100; - - // TR / SR algorithms specific metadata - if ( - network.rarityAlgorithm == rarityAlgorithms.TraitRarity || - network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts - // ps: the only difference being that attributeRarityNormed is calculated only once - const totalLayersCnt = Object.keys(traitOccurances).length; - const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; - asset[1].attributeFrequency = - asset[1].attributeOccurrence / metadataList.length; - asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; - asset[1].attributeRarity = - metadataList.length / asset[1].attributeOccurrence; - asset[1].attributeRarityNormed = - asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); - } - }); - }); - - // calculate rarity for each item/NFT - if (network.rarityAlgorithm == rarityAlgorithms.none) { - return; - } else if (network.rarityAlgorithm == rarityAlgorithms.JaccardDistances) { - let z = []; - let avg = []; - - // calculate z(i,j) and avg(i) - for (let i = 0; i < metadataList.length; i++) { - for (let j = 0; j < metadataList.length; j++) { - if (i == j) continue; - - if (z[i] == null) { - z[i] = []; - } - - if (z[i][j] == null || z[j][i] == null) { - const commonTraitsCnt = getObjectCommonCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - const uniqueTraitsCnt = getObjectUniqueCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - - z[i][j] = commonTraitsCnt / uniqueTraitsCnt; - } - } - - // ps: length-1 because there's always an empty cell in matrix, where i == j - avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); - } - - // calculate z(i) - let jd = []; - let avgMax = Math.max(...avg); - let avgMin = Math.min(...avg); - - for (let i = 0; i < metadataList.length; i++) { - jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; - } - - const jd_asc = [...jd].sort(function (a, b) { - return a - b; - }); - - // add JD rarity data to NFT/item - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity = { - score: jd[i], - }; - if (network.includeRank) { - metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); - } - } - } else if ( - network.rarityAlgorithm == rarityAlgorithms.TraitRarity || - network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - metadataList.forEach((item) => { - item.rarity = { - usedTraitsCount: item.attributes.length, - }; - // TR specific - if ( - network.rarityAlgorithm == rarityAlgorithms.TraitRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - item.rarity.avgRarity = 0; - item.rarity.rarityScore = 0; - item.rarity.rarityScoreNormed = 0; - } - // SR specific - if ( - network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - item.rarity.statRarity = 1; - } - - item.attributes.forEach((a) => { - const attributeData = rarityObject[a.trait_type][a.value]; - if ( - network.rarityAlgorithm == rarityAlgorithms.TraitRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - item.rarity.avgRarity += attributeData.attributeFrequency; - item.rarity.rarityScore += attributeData.attributeRarity; - item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; - } - // SR specific - if ( - network.rarityAlgorithm == rarityAlgorithms.StatisticalRarity || - network.rarityAlgorithm == rarityAlgorithms.TraitAndStatisticalRarity - ) { - item.rarity.statRarity *= attributeData.attributeFrequency; - } - }); - }); - - // add rarity rank - if (network.includeRank) { - const metadataList_asc = [...metadataList].sort(function (a, b) { - return ( - a.rarity.rarityScore - b.rarity.rarityScore ?? - b.rarity.statRarity - a.rarity.statRarity - ); - }); - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity.rank = - metadataList.length - metadataList_asc.indexOf(metadataList[i]); - } - } - } -}; - const addAttributes = (_element) => { let selectedElement = _element.layer.selectedElement; attributesList.push({ @@ -509,7 +273,7 @@ const createDna = (_layers) => { const writeMetaData = (_data) => { fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix ?? ""}${network.metadataFileName}`, + `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, _data ); }; @@ -525,7 +289,7 @@ const saveIndividualMetadataFiles = (abstractedIndexes) => { ) : null; fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix ?? ""}${ + `${buildDir}/${network.jsonDirPrefix}${ item.edition || abstractedIndexes[idx] }.json`, JSON.stringify(item, null, 2) @@ -591,9 +355,7 @@ const startCreating = async () => { hashlipsGiffer = new HashlipsGiffer( canvas, ctx, - `${buildDir}/${network.mediaDirPrefix ?? ""}${ - abstractedIndexes[0] - }.gif`, + `${buildDir}/${network.mediaDirPrefix}${abstractedIndexes[0]}.gif`, gif.repeat, gif.quality, gif.delay @@ -620,7 +382,10 @@ const startCreating = async () => { ? console.log("Editions left to create: ", abstractedIndexes) : null; saveImage(abstractedIndexes[0]); - addMetadata(newDna, abstractedIndexes[0]); + metadataList.push( + createMetadataItem(attributesList, newDna, abstractedIndexes[0]) + ); + attributesList = []; console.log( `Created edition: ${abstractedIndexes[0]}, with DNA: ${sha1( newDna @@ -644,20 +409,21 @@ const startCreating = async () => { layerConfigIndex++; } - // build rarity - if (network.metadataType == metadataTypes.rarities) { - addRarityMetadata(); + // calculate rarities (if needed) & save _metadata.json file + if (network.metadataType === METADATA.basic) { + writeMetaData(JSON.stringify(metadataList, null, 2)); + } else if (network.metadataType === METADATA.rarities) { + // calculate rarity for traits/layers & attributes/assets + const rarityObject = getGeneralRarity(metadataList); + if (network.rarityAlgorithm !== RARITY.none) { + // calculate rarity for all items/NFTs + metadataList = getItemsRarity(metadataList, rarityObject); + } + writeMetaData(JSON.stringify(rarityObject, null, 2)); } // save individual metadata files saveIndividualMetadataFiles(abstractedIndexesBackup); - - // save metadata.json - if (network.metadataType == metadataTypes.basic) { - writeMetaData(JSON.stringify(metadataList, null, 2)); - } else { - writeMetaData(JSON.stringify(rarityObject, null, 2)); - } }; module.exports = { startCreating, buildSetup, getElements }; diff --git a/utils/common.js b/utils/common.js index a469a198a..7d5330cd7 100644 --- a/utils/common.js +++ b/utils/common.js @@ -1,38 +1,28 @@ const basePath = process.cwd(); const fs = require("fs"); const { network } = require(`${basePath}/src/config.js`); -const { metadataTypes } = require(`${basePath}/constants/network.js`); +const { METADATA } = require(`${basePath}/constants/metadata.js`); // get metadata of all generated NFTs/items const getMetadataItems = () => { - if (network.metadataType == metadataTypes.basic) { + if (network.metadataType === METADATA.basic) { // get metadata from the _metadata.json file return JSON.parse( fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${ - network.metadataFileName - }` + `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` ) ); } else { // get metadata from the individual metadata files - let rawData = []; const jsonFilePattern = /^\d+.(json)$/i; - const files = fs.readdirSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}` - ); - files.forEach((file) => { - if (file.match(jsonFilePattern)) { - rawData.push( - JSON.parse( - fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${file}` - ) - ) - ); - } - }); - return rawData; + const files = fs.readdirSync(`${basePath}/build/${network.jsonDirPrefix}`); + return files + .filter((file) => file.match(jsonFilePattern)) + .map((file) => + JSON.parse( + fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${file}`) + ) + ); } }; diff --git a/utils/metadata.js b/utils/metadata.js new file mode 100644 index 000000000..671f4dd7b --- /dev/null +++ b/utils/metadata.js @@ -0,0 +1,73 @@ +const basePath = process.cwd(); +const { NETWORK } = require(`${basePath}/constants/network.js`); +const { METADATA } = require(`${basePath}/constants/metadata.js`); +const { RARITY } = require(`${basePath}/constants/rarity.js`); +const sha1 = require(`${basePath}/node_modules/sha1`); +const { + baseUri, + description, + extraMetadata, + namePrefix, + network, + solanaMetadata, +} = require(`${basePath}/src/config.js`); + +// create metadata item with the provided data +const createMetadataItem = (attributesList, _dna, _edition) => { + let tempMetadata = {}; + + switch (network) { + case NETWORK.egld: { + tempMetadata = { + description: description, + dna: sha1(_dna), + ...extraMetadata, + attributes: attributesList, + rarities: {}, + compiler: "HashLips Art Engine", + }; + } + case NETWORK.eth: { + tempMetadata = { + name: `${namePrefix} #${_edition}`, + description: description, + image: `${baseUri}/${_edition}.png`, + dna: sha1(_dna), + edition: _edition, + date: Date.now(), + attributes: attributesList, + ...extraMetadata, + compiler: "HashLips Art Engine", + }; + break; + } + case NETWORK.sol: { + tempMetadata = { + name: `${namePrefix} #${_edition}`, + symbol: solanaMetadata.symbol, + description: description, + seller_fee_basis_points: solanaMetadata.seller_fee_basis_points, + image: `${_edition}.png`, + external_url: solanaMetadata.external_url, + edition: _edition, + ...extraMetadata, + attributes: attributesList, + properties: { + files: [ + { + uri: `${_edition}.png`, + type: "image/png", + }, + ], + category: "image", + creators: solanaMetadata.creators, + }, + }; + break; + } + } + + return tempMetadata; +}; + +module.exports = { createMetadataItem }; diff --git a/utils/rarity.js b/utils/rarity.js index f4efcbf4f..283643f94 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -1,89 +1,212 @@ const basePath = process.cwd(); -const fs = require("fs"); -const { exit } = require("process"); -const { metadataTypes } = require("../constants/network.js"); -const { network } = require("../src/config.js"); -const layersDir = `${basePath}/layers`; - -const { layerConfigurations } = require(`${basePath}/src/config.js`); -const { getMetadataItems } = require(`${basePath}/utils/common.js`); - -const { getElements } = require("../src/main.js"); - -// read json data -let data = getMetadataItems(); -let editionSize = data.length; - -let rarityData = []; - -// intialize layers to chart -console.log("layerconfig", layerConfigurations); -layerConfigurations.forEach((config) => { - let layers = config.layersOrder; - - layers.forEach((layer) => { - // get elements for each layer - let elementsForLayer = []; - let elements = getElements(`${layersDir}/${layer.name}/`); - elements.forEach((element) => { - // just get name and weight for each element - let rarityDataElement = { - trait: element.name, - weight: element.weight.toFixed(0), - occurrence: 0, // initialize at 0 - }; - elementsForLayer.push(rarityDataElement); +const { NETWORK } = require(`${basePath}/constants/network.js`); +const { METADATA } = require(`${basePath}/constants/metadata.js`); +const { RARITY } = require(`${basePath}/constants/rarity.js`); +const { network } = require(`${basePath}/src/config.js`); +const { + getObjectCommonCnt, + getObjectUniqueCnt, +} = require(`${basePath}/utils/common.js`); + +// calculates rarity for each trait/layer & attribute/asset +const getGeneralRarity = (metadataList) => { + let rarityObject = {}; + let traitOccurances = []; + let totalAttributesCnt = 0; + + // count occurrences for all traits/layers & attributes/assets + metadataList.forEach((item) => { + item.attributes.forEach((a) => { + if (rarityObject[a.trait_type] != null) { + if (rarityObject[a.trait_type][a.value] != null) { + rarityObject[a.trait_type][a.value].attributeOccurrence++; + } else { + rarityObject[a.trait_type][a.value] = { attributeOccurrence: 1 }; + totalAttributesCnt++; + } + + traitOccurances[a.trait_type]++; + } else { + rarityObject[a.trait_type] = { [a.value]: { attributeOccurrence: 1 } }; + traitOccurances[a.trait_type] = 1; + totalAttributesCnt++; + } }); - let layerName = - layer.options?.["displayName"] != undefined - ? layer.options?.["displayName"] - : layer.name; - // don't include duplicate layers - if (!rarityData.includes(layer.name)) { - // add elements for each layer to chart - rarityData[layerName] = elementsForLayer; - } }); -}); - -// fill up rarity chart with occurrences from metadata -data.forEach((element) => { - let attributes = element.attributes; - attributes.forEach((attribute) => { - let traitType = attribute.trait_type; - let value = attribute.value; - - let rarityDataTraits = rarityData[traitType]; - rarityDataTraits.forEach((rarityDataTrait) => { - if (rarityDataTrait.trait == value) { - // keep track of occurrences - rarityDataTrait.occurrence++; + + // general rarity metadata for all traits/layers & attributes/assets + Object.entries(rarityObject).forEach((entry) => { + const layer = entry[0]; + const assets = entry[1]; + + Object.entries(assets).forEach((asset) => { + // trait/layer related + asset[1].traitOccurance = traitOccurances[layer]; + asset[1].traitOccurancePercentage = + (metadataList.length / asset[1].traitOccurance) * 100; + + // attribute/asset related + asset[1].attributeOccurrencePercentage = + (asset[1].attributeOccurrence / metadataList.length) * 100; + + // TR / SR algorithms specific metadata + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts + // ps: the only difference being that attributeRarityNormed is calculated only once + const totalLayersCnt = Object.keys(traitOccurances).length; + const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; + asset[1].attributeFrequency = + asset[1].attributeOccurrence / metadataList.length; + asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; + asset[1].attributeRarity = + metadataList.length / asset[1].attributeOccurrence; + asset[1].attributeRarityNormed = + asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); } }); }); -}); - -// convert occurrences to occurrences string -for (var layer in rarityData) { - for (var attribute in rarityData[layer]) { - // get chance - let chance = ( - (rarityData[layer][attribute].occurrence / editionSize) * - 100 - ).toFixed(2); - - // show two decimal places in percent - rarityData[layer][ - attribute - ].occurrence = `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; + + return rarityObject; +}; + +// calculates rarity for all items/NFT +const getItemsRarity = (metadataList, rarityObject) => { + switch (network.rarityAlgorithm) { + case RARITY.none: { + return; + } + case RARITY.JaccardDistances: { + return getItemsRarity_JaccardDistances(metadataList); + } + case RARITY.TraitRarity: + case RARITY.StatisticalRarity: + case RARITY.TraitAndStatisticalRarity: { + return getItemsRarity_TSR(metadataList, rarityObject); + } + default: + break; + } +}; + +// calculates rarity for all items/NFT using Jaccard Distances algorithm +const getItemsRarity_JaccardDistances = (metadataList) => { + let z = []; + let avg = []; + + // calculate z(i,j) and avg(i) + for (let i = 0; i < metadataList.length; i++) { + for (let j = 0; j < metadataList.length; j++) { + if (i == j) continue; + + if (z[i] == null) { + z[i] = []; + } + + if (z[i][j] == null || z[j][i] == null) { + const commonTraitsCnt = getObjectCommonCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + const uniqueTraitsCnt = getObjectUniqueCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + + z[i][j] = commonTraitsCnt / uniqueTraitsCnt; + } + } + + // ps: length-1 because there's always an empty cell in matrix, where i == j + avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); + } + + // calculate z(i) + let jd = []; + let avgMax = Math.max(...avg); + let avgMin = Math.min(...avg); + + for (let i = 0; i < metadataList.length; i++) { + jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; + } + + const jd_asc = [...jd].sort(function (a, b) { + return a - b; + }); + + // add JD rarity data to NFT/item + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity = { + score: jd[i], + }; + if (network.includeRank) { + metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); + } } -} + return metadataList; +}; + +// calculates rarity for all items/NFT using Trait/Statistical rarity algorithm(s) +const getItemsRarity_TSR = (metadataList, rarityObject) => { + metadataList.forEach((item) => { + item.rarity = { + usedTraitsCount: item.attributes.length, + }; + // TR specific + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.avgRarity = 0; + item.rarity.rarityScore = 0; + item.rarity.rarityScoreNormed = 0; + } + // SR specific + if ( + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.statRarity = 1; + } -// print out rarity data -for (var layer in rarityData) { - console.log(`Trait type: ${layer}`); - for (var trait in rarityData[layer]) { - console.log(rarityData[layer][trait]); + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.avgRarity += attributeData.attributeFrequency; + item.rarity.rarityScore += attributeData.attributeRarity; + item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + } + // SR specific + if ( + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.statRarity *= attributeData.attributeFrequency; + } + }); + }); + + // add rarity rank + if (network.includeRank) { + const metadataList_asc = [...metadataList].sort(function (a, b) { + return ( + a.rarity.rarityScore - b.rarity.rarityScore ?? + b.rarity.statRarity - a.rarity.statRarity + ); + }); + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity.rank = + metadataList.length - metadataList_asc.indexOf(metadataList[i]); + } } - console.log(); -} + + return metadataList; +}; + +module.exports = { getGeneralRarity, getItemsRarity }; diff --git a/utils/generate_metadata.js b/utils/scripts/generate_metadata.js similarity index 98% rename from utils/generate_metadata.js rename to utils/scripts/generate_metadata.js index 458490ed9..b50c0e23d 100644 --- a/utils/generate_metadata.js +++ b/utils/scripts/generate_metadata.js @@ -2,6 +2,7 @@ const fs = require("fs"); const path = require("path"); const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); +const { network } = require(`${basePath}/src/config.js`); const buildDir = `${basePath}/build/${network.jsonDirPath}`; const inputDir = `${basePath}/build/${network.jsonDirPath}`; const { @@ -11,7 +12,6 @@ const { baseUri, } = require(`${basePath}/src/config.js`); const console = require("console"); -const { network } = require("../src/config"); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); const metadataList = []; diff --git a/utils/pixelate.js b/utils/scripts/pixelate.js similarity index 97% rename from utils/pixelate.js rename to utils/scripts/pixelate.js index 50a099b6e..29178085b 100644 --- a/utils/pixelate.js +++ b/utils/scripts/pixelate.js @@ -4,7 +4,7 @@ const { createCanvas, loadImage } = require("canvas"); const basePath = process.cwd(); const { format, network, pixelFormat } = require(`${basePath}/src/config.js`); const buildDir = `${basePath}/build/pixel_images`; -const inputDir = `${basePath}/build/${network.mediaDirPrefix ?? ""}`; +const inputDir = `${basePath}/build/${network.mediaDirPrefix}`; const console = require("console"); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); diff --git a/utils/preview.js b/utils/scripts/preview.js similarity index 95% rename from utils/preview.js rename to utils/scripts/preview.js index 52dce470c..fd48ab608 100644 --- a/utils/preview.js +++ b/utils/scripts/preview.js @@ -33,8 +33,8 @@ const saveProjectPreviewImage = async (_data) => { for (let index = 0; index < _data.length; index++) { const nft = _data[index]; await loadImage( - `${buildDir}/${network.mediaDirPrefix ?? ""}${ - network.mediaFilePrefix ?? "" + `${buildDir}/${network.mediaDirPrefix}${ + network.mediaFilePrefix }${nft.edition}.png` ).then((image) => { previewCtx.drawImage( diff --git a/utils/preview_gif.js b/utils/scripts/preview_gif.js similarity index 97% rename from utils/preview_gif.js rename to utils/scripts/preview_gif.js index 078fc9f75..d52a0535a 100644 --- a/utils/preview_gif.js +++ b/utils/scripts/preview_gif.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const fs = require("fs"); const { createCanvas, loadImage } = require("canvas"); const buildDir = `${basePath}/build`; -const imageDir = `${buildDir}/${network.mediaDirPrefix ?? ""}`; +const imageDir = `${buildDir}/${network.mediaDirPrefix}`; const { format, preview_gif } = require(`${basePath}/src/config.js`); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); diff --git a/utils/scripts/rarity.js b/utils/scripts/rarity.js new file mode 100644 index 000000000..bdbf8f111 --- /dev/null +++ b/utils/scripts/rarity.js @@ -0,0 +1,89 @@ +const basePath = process.cwd(); +const fs = require("fs"); +const { exit } = require("process"); +const { METADATA } = require(`${basePath}/constants/metadata.js`); +const { network } = require(`${basePath}/src/config.js`); +const layersDir = `${basePath}/layers`; + +const { layerConfigurations } = require(`${basePath}/src/config.js`); +const { getMetadataItems } = require(`${basePath}/utils/common.js`); + +const { getElements } = require(`${basePath}/src/main.js`); + +// read json data +let data = getMetadataItems(); +let editionSize = data.length; + +let rarityData = []; + +// intialize layers to chart +console.log("layerconfig", layerConfigurations); +layerConfigurations.forEach((config) => { + let layers = config.layersOrder; + + layers.forEach((layer) => { + // get elements for each layer + let elementsForLayer = []; + let elements = getElements(`${layersDir}/${layer.name}/`); + elements.forEach((element) => { + // just get name and weight for each element + let rarityDataElement = { + trait: element.name, + weight: element.weight.toFixed(0), + occurrence: 0, // initialize at 0 + }; + elementsForLayer.push(rarityDataElement); + }); + let layerName = + layer.options?.["displayName"] != undefined + ? layer.options?.["displayName"] + : layer.name; + // don't include duplicate layers + if (!rarityData.includes(layer.name)) { + // add elements for each layer to chart + rarityData[layerName] = elementsForLayer; + } + }); +}); + +// fill up rarity chart with occurrences from metadata +data.forEach((element) => { + let attributes = element.attributes; + attributes.forEach((attribute) => { + let traitType = attribute.trait_type; + let value = attribute.value; + + let rarityDataTraits = rarityData[traitType]; + rarityDataTraits.forEach((rarityDataTrait) => { + if (rarityDataTrait.trait == value) { + // keep track of occurrences + rarityDataTrait.occurrence++; + } + }); + }); +}); + +// convert occurrences to occurrences string +for (var layer in rarityData) { + for (var attribute in rarityData[layer]) { + // get chance + let chance = ( + (rarityData[layer][attribute].occurrence / editionSize) * + 100 + ).toFixed(2); + + // show two decimal places in percent + rarityData[layer][ + attribute + ].occurrence = `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; + } +} + +// print out rarity data +for (var layer in rarityData) { + console.log(`Trait type: ${layer}`); + for (var trait in rarityData[layer]) { + console.log(rarityData[layer][trait]); + } + console.log(); +} diff --git a/utils/update_info.js b/utils/scripts/update_info.js similarity index 77% rename from utils/update_info.js rename to utils/scripts/update_info.js index 8d1e8a4d2..d058f9303 100644 --- a/utils/update_info.js +++ b/utils/scripts/update_info.js @@ -1,7 +1,7 @@ const basePath = process.cwd(); const { NETWORK } = require(`${basePath}/constants/network.js`); const fs = require("fs"); -const { metadataTypes } = require("../constants/network"); +const { METADATA } = require(`${basePath}/constants/metadata`); const { getMetadataItems } = require(`${basePath}/utils/common.js`); const { @@ -19,36 +19,36 @@ let idx = network.startIdx; data.forEach((item) => { // general metadata item.description = description; - if (network.metadataType != metadataTypes.basic) { + if (network.metadataType != METADATA.basic) { item.name = `${namePrefix} #${idx++}`; } else { item.name = `${namePrefix} #${item.edition}`; } // custom metadata - if (network == NETWORK.eth) { + if (network === NETWORK.eth) { item.image = `${baseUri}/${item.edition}.png`; } - if (network == NETWORK.sol) { + if (network === NETWORK.sol) { item.creators = solanaMetadata.creators; } fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${item.edition}.json`, + `${basePath}/build/${network.jsonDirPrefix}${item.edition}.json`, JSON.stringify(item, null, 2) ); }); -if (network.metadataType == metadataTypes.basic) { +if (network.metadataType === METADATA.basic) { fs.writeFileSync( - `${basePath}/build/${network.jsonDirPrefix ?? ""}${ + `${basePath}/build/${network.jsonDirPrefix}${ network.metadataFileName }`, JSON.stringify(data, null, 2) ); } -if (network == NETWORK.sol) { +if (network === NETWORK.sol) { console.log(`Updated description for images to ===> ${description}`); console.log(`Updated name prefix for images to ===> ${namePrefix}`); console.log( From ffba6512717c57e8f345ecc9286e1819a31e26bb Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 26 Apr 2022 00:57:14 +0300 Subject: [PATCH 12/19] remove common.js; restructure project; improve some code --- constants/metadata.js | 4 +- package.json | 12 +- src/main.js | 8 +- {utils => src}/metadata.js | 27 ++- src/rarity.js | 245 ++++++++++++++++++++ utils/common.js | 76 ------ utils/{scripts => }/generate_metadata.js | 0 utils/{scripts => }/pixelate.js | 0 utils/{scripts => }/preview.js | 2 +- utils/{scripts => }/preview_gif.js | 0 utils/rarity.js | 279 +++++++---------------- utils/scripts/rarity.js | 89 -------- utils/{scripts => }/update_info.js | 2 +- 13 files changed, 360 insertions(+), 384 deletions(-) rename {utils => src}/metadata.js (69%) create mode 100644 src/rarity.js delete mode 100644 utils/common.js rename utils/{scripts => }/generate_metadata.js (100%) rename utils/{scripts => }/pixelate.js (100%) rename utils/{scripts => }/preview.js (96%) rename utils/{scripts => }/preview_gif.js (100%) delete mode 100644 utils/scripts/rarity.js rename utils/{scripts => }/update_info.js (96%) diff --git a/constants/metadata.js b/constants/metadata.js index 13059fa8d..b517fc968 100644 --- a/constants/metadata.js +++ b/constants/metadata.js @@ -1,8 +1,8 @@ const METADATA = { - // metadata file will contain all individual metadata files (common for eth$, sol$) + // metadata file will contain all individual metadata files (common for eth, sol) // no rarities at all basic: 0, - // metadata file will contain only rarity data for traits & attributes (common for egld$) + // metadata file will contain only rarity data for traits & attributes (common for egld) // if rarityAlgorithm provided, individual metadata files will also contain rarity data rarities: 1, }; diff --git a/package.json b/package.json index 203793880..141abf44e 100644 --- a/package.json +++ b/package.json @@ -14,12 +14,12 @@ "scripts": { "build": "node index.js", "generate": "node index.js", - "rarity": "node utils/scripts/rarity.js", - "preview": "node utils/scripts/preview.js", - "pixelate": "node utils/scripts/pixelate.js", - "update_info": "node utils/scripts/update_info.js", - "preview_gif": "node utils/scripts/preview_gif.js", - "generate_metadata": "node utils/scripts/generate_metadata.js" + "rarity": "node utils/rarity.js", + "preview": "node utils/preview.js", + "pixelate": "node utils/pixelate.js", + "update_info": "node utils/update_info.js", + "preview_gif": "node utils/preview_gif.js", + "generate_metadata": "node utils/generate_metadata.js" }, "author": "Daniel Eugene Botha (HashLips)", "license": "MIT", diff --git a/src/main.js b/src/main.js index a98dbcad0..41608519f 100644 --- a/src/main.js +++ b/src/main.js @@ -24,15 +24,11 @@ const { solanaMetadata, gif, } = require(`${basePath}/src/config.js`); -const { createMetadataItem } = require(`${basePath}/utils/metadata.js`); +const { createMetadataItem } = require(`${basePath}/src/metadata.js`); const { getGeneralRarity, getItemsRarity, -} = require(`${basePath}/utils/rarity.js`); -const { - getObjectCommonCnt, - getObjectUniqueCnt, -} = require(`${basePath}/utils/common.js`); +} = require(`${basePath}/src/rarity.js`); const canvas = createCanvas(format.width, format.height); const ctx = canvas.getContext("2d"); ctx.imageSmoothingEnabled = format.smoothing; diff --git a/utils/metadata.js b/src/metadata.js similarity index 69% rename from utils/metadata.js rename to src/metadata.js index 671f4dd7b..27b1b5192 100644 --- a/utils/metadata.js +++ b/src/metadata.js @@ -1,7 +1,7 @@ const basePath = process.cwd(); const { NETWORK } = require(`${basePath}/constants/network.js`); const { METADATA } = require(`${basePath}/constants/metadata.js`); -const { RARITY } = require(`${basePath}/constants/rarity.js`); +const fs = require("fs"); const sha1 = require(`${basePath}/node_modules/sha1`); const { baseUri, @@ -70,4 +70,27 @@ const createMetadataItem = (attributesList, _dna, _edition) => { return tempMetadata; }; -module.exports = { createMetadataItem }; +// get metadata of all generated NFTs/items +const getMetadataItems = () => { + if (network.metadataType === METADATA.basic) { + // get metadata from the _metadata.json file + return JSON.parse( + fs.readFileSync( + `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` + ) + ); + } else { + // get metadata from the individual metadata files + const jsonFilePattern = /^\d+.(json)$/i; + const files = fs.readdirSync(`${basePath}/build/${network.jsonDirPrefix}`); + return files + .filter((file) => file.match(jsonFilePattern)) + .map((file) => + JSON.parse( + fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${file}`) + ) + ); + } +}; + +module.exports = { createMetadataItem, getMetadataItems }; diff --git a/src/rarity.js b/src/rarity.js new file mode 100644 index 000000000..8fdfc339f --- /dev/null +++ b/src/rarity.js @@ -0,0 +1,245 @@ +const basePath = process.cwd(); +const { RARITY } = require(`${basePath}/constants/rarity.js`); +const { network } = require(`${basePath}/src/config.js`); + +// calculates rarity for each trait/layer & attribute/asset +const getGeneralRarity = (metadataList) => { + let rarityObject = {}; + let traitOccurances = []; + let totalAttributesCnt = 0; + + // count occurrences for all traits/layers & attributes/assets + metadataList.forEach((item) => { + item.attributes.forEach((a) => { + if (rarityObject[a.trait_type] != null) { + if (rarityObject[a.trait_type][a.value] != null) { + rarityObject[a.trait_type][a.value].attributeOccurrence++; + } else { + rarityObject[a.trait_type][a.value] = { attributeOccurrence: 1 }; + totalAttributesCnt++; + } + + traitOccurances[a.trait_type]++; + } else { + rarityObject[a.trait_type] = { [a.value]: { attributeOccurrence: 1 } }; + traitOccurances[a.trait_type] = 1; + totalAttributesCnt++; + } + }); + }); + + // general rarity metadata for all traits/layers & attributes/assets + Object.entries(rarityObject).forEach((entry) => { + const layer = entry[0]; + const assets = entry[1]; + + Object.entries(assets).forEach((asset) => { + // trait/layer related + asset[1].traitOccurance = traitOccurances[layer]; + asset[1].traitOccurancePercentage = + (metadataList.length / asset[1].traitOccurance) * 100; + + // attribute/asset related + asset[1].attributeOccurrencePercentage = + (asset[1].attributeOccurrence / metadataList.length) * 100; + + // TR / SR algorithms specific metadata + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts + // ps: the only difference being that attributeRarityNormed is calculated only once + const totalLayersCnt = Object.keys(traitOccurances).length; + const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; + asset[1].attributeFrequency = + asset[1].attributeOccurrence / metadataList.length; + asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; + asset[1].attributeRarity = + metadataList.length / asset[1].attributeOccurrence; + asset[1].attributeRarityNormed = + asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); + } + }); + }); + + return rarityObject; +}; + +// calculates rarity for all items/NFT +const getItemsRarity = (metadataList, rarityObject) => { + switch (network.rarityAlgorithm) { + case RARITY.none: { + return; + } + case RARITY.JaccardDistances: { + return getItemsRarity_JaccardDistances(metadataList); + } + case RARITY.TraitRarity: + case RARITY.StatisticalRarity: + case RARITY.TraitAndStatisticalRarity: { + return getItemsRarity_TSR(metadataList, rarityObject); + } + default: + break; + } +}; + +// calculates rarity for all items/NFT using Jaccard Distances algorithm +const getItemsRarity_JaccardDistances = (metadataList) => { + let z = []; + let avg = []; + + // calculate z(i,j) and avg(i) + for (let i = 0; i < metadataList.length; i++) { + for (let j = 0; j < metadataList.length; j++) { + if (i == j) continue; + + if (z[i] == null) { + z[i] = []; + } + + if (z[i][j] == null || z[j][i] == null) { + const commonTraitsCnt = getObjectCommonCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + const uniqueTraitsCnt = getObjectUniqueCnt( + metadataList[i].attributes, + metadataList[j].attributes + ); + + z[i][j] = commonTraitsCnt / uniqueTraitsCnt; + } + } + + // ps: length-1 because there's always an empty cell in matrix, where i == j + avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); + } + + // calculate z(i) + let jd = []; + let avgMax = Math.max(...avg); + let avgMin = Math.min(...avg); + + for (let i = 0; i < metadataList.length; i++) { + jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; + } + + const jd_asc = [...jd].sort(function (a, b) { + return a - b; + }); + + // add JD rarity data to NFT/item + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity = { + score: jd[i], + }; + if (network.includeRank) { + metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); + } + } + return metadataList; +}; + +// calculates rarity for all items/NFT using Trait/Statistical rarity algorithm(s) +const getItemsRarity_TSR = (metadataList, rarityObject) => { + metadataList.forEach((item) => { + item.rarity = { + usedTraitsCount: item.attributes.length, + }; + // TR specific + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.avgRarity = 0; + item.rarity.rarityScore = 0; + item.rarity.rarityScoreNormed = 0; + } + // SR specific + if ( + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.statRarity = 1; + } + + item.attributes.forEach((a) => { + const attributeData = rarityObject[a.trait_type][a.value]; + if ( + network.rarityAlgorithm === RARITY.TraitRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.avgRarity += attributeData.attributeFrequency; + item.rarity.rarityScore += attributeData.attributeRarity; + item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; + } + // SR specific + if ( + network.rarityAlgorithm === RARITY.StatisticalRarity || + network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + ) { + item.rarity.statRarity *= attributeData.attributeFrequency; + } + }); + }); + + // add rarity rank + if (network.includeRank) { + const metadataList_asc = [...metadataList].sort(function (a, b) { + return ( + a.rarity.rarityScore - b.rarity.rarityScore ?? + b.rarity.statRarity - a.rarity.statRarity + ); + }); + for (let i = 0; i < metadataList.length; i++) { + metadataList[i].rarity.rank = + metadataList.length - metadataList_asc.indexOf(metadataList[i]); + } + } + + return metadataList; +}; + +// get common elements counter of 2 arrays +const getArrayCommonCnt = (arr1, arr2) => { + return arr1.filter((e) => { + return arr2.includes(e); + }).length; +}; + +// get unique elements counter of 2 arrays +const getArrayUniqueCnt = (arr1, arr2) => { + return [...new Set(arr1.concat(arr2))].length; +}; + +// get common elements counter of 2 objects +const getObjectCommonCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + for (const [key, value] of Object.entries(obj1)) { + arr1.push(JSON.stringify(value)); + } + for (const [key, value] of Object.entries(obj2)) { + arr2.push(JSON.stringify(value)); + } + + return getArrayCommonCnt(arr1, arr2); +}; + +// get unique elements counter of 2 objects +const getObjectUniqueCnt = (obj1, obj2) => { + let arr1 = []; + let arr2 = []; + for (const [key, value] of Object.entries(obj1)) { + arr1.push(JSON.stringify(value)); + } + for (const [key, value] of Object.entries(obj2)) { + arr2.push(JSON.stringify(value)); + } + return getArrayUniqueCnt(arr1, arr2); +}; + +module.exports = { getGeneralRarity, getItemsRarity }; diff --git a/utils/common.js b/utils/common.js deleted file mode 100644 index 7d5330cd7..000000000 --- a/utils/common.js +++ /dev/null @@ -1,76 +0,0 @@ -const basePath = process.cwd(); -const fs = require("fs"); -const { network } = require(`${basePath}/src/config.js`); -const { METADATA } = require(`${basePath}/constants/metadata.js`); - -// get metadata of all generated NFTs/items -const getMetadataItems = () => { - if (network.metadataType === METADATA.basic) { - // get metadata from the _metadata.json file - return JSON.parse( - fs.readFileSync( - `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` - ) - ); - } else { - // get metadata from the individual metadata files - const jsonFilePattern = /^\d+.(json)$/i; - const files = fs.readdirSync(`${basePath}/build/${network.jsonDirPrefix}`); - return files - .filter((file) => file.match(jsonFilePattern)) - .map((file) => - JSON.parse( - fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${file}`) - ) - ); - } -}; - -// get common elements counter of 2 arrays -const getArrayCommonCnt = (arr1, arr2) => { - var cnt = 0; - for (var i = 0; i < arr1.length; ++i) { - for (var j = 0; j < arr2.length; ++j) { - if (arr1[i] == arr2[j]) { - cnt++; - } - } - } - return cnt; -}; -// get unique elements counter of 2 arrays -const getArrayUniqueCnt = (arr1, arr2) => { - return [...new Set(arr1.concat(arr2))].length; -}; -// get common elements counter of 2 objects -const getObjectCommonCnt = (obj1, obj2) => { - let arr1 = []; - let arr2 = []; - Object.entries(obj1).forEach((entry) => { - arr1.push(JSON.stringify(entry[1])); - }); - Object.entries(obj2).forEach((entry) => { - arr2.push(JSON.stringify(entry[1])); - }); - return getArrayCommonCnt(arr1, arr2); -}; -// get unique elements counter of 2 objects -const getObjectUniqueCnt = (obj1, obj2) => { - let arr1 = []; - let arr2 = []; - Object.entries(obj1).forEach((entry) => { - arr1.push(JSON.stringify(entry[1])); - }); - Object.entries(obj2).forEach((entry) => { - arr2.push(JSON.stringify(entry[1])); - }); - return getArrayUniqueCnt(arr1, arr2); -}; - -module.exports = { - getMetadataItems, - getArrayCommonCnt, - getArrayUniqueCnt, - getObjectCommonCnt, - getObjectUniqueCnt, -}; diff --git a/utils/scripts/generate_metadata.js b/utils/generate_metadata.js similarity index 100% rename from utils/scripts/generate_metadata.js rename to utils/generate_metadata.js diff --git a/utils/scripts/pixelate.js b/utils/pixelate.js similarity index 100% rename from utils/scripts/pixelate.js rename to utils/pixelate.js diff --git a/utils/scripts/preview.js b/utils/preview.js similarity index 96% rename from utils/scripts/preview.js rename to utils/preview.js index fd48ab608..fafce19da 100644 --- a/utils/scripts/preview.js +++ b/utils/preview.js @@ -4,7 +4,7 @@ const { createCanvas, loadImage } = require("canvas"); const buildDir = `${basePath}/build`; const { network, preview } = require(`${basePath}/src/config.js`); -const { getMetadataItems } = require(`${basePath}/utils/common.js`); +const { getMetadataItems } = require(`${basePath}/src/metadata.js`); // read json data const metadataList = getMetadataItems(); diff --git a/utils/scripts/preview_gif.js b/utils/preview_gif.js similarity index 100% rename from utils/scripts/preview_gif.js rename to utils/preview_gif.js diff --git a/utils/rarity.js b/utils/rarity.js index 283643f94..361e553ae 100644 --- a/utils/rarity.js +++ b/utils/rarity.js @@ -1,212 +1,89 @@ const basePath = process.cwd(); -const { NETWORK } = require(`${basePath}/constants/network.js`); +const fs = require("fs"); +const { exit } = require("process"); const { METADATA } = require(`${basePath}/constants/metadata.js`); -const { RARITY } = require(`${basePath}/constants/rarity.js`); const { network } = require(`${basePath}/src/config.js`); -const { - getObjectCommonCnt, - getObjectUniqueCnt, -} = require(`${basePath}/utils/common.js`); - -// calculates rarity for each trait/layer & attribute/asset -const getGeneralRarity = (metadataList) => { - let rarityObject = {}; - let traitOccurances = []; - let totalAttributesCnt = 0; - - // count occurrences for all traits/layers & attributes/assets - metadataList.forEach((item) => { - item.attributes.forEach((a) => { - if (rarityObject[a.trait_type] != null) { - if (rarityObject[a.trait_type][a.value] != null) { - rarityObject[a.trait_type][a.value].attributeOccurrence++; - } else { - rarityObject[a.trait_type][a.value] = { attributeOccurrence: 1 }; - totalAttributesCnt++; - } - - traitOccurances[a.trait_type]++; - } else { - rarityObject[a.trait_type] = { [a.value]: { attributeOccurrence: 1 } }; - traitOccurances[a.trait_type] = 1; - totalAttributesCnt++; - } - }); - }); - - // general rarity metadata for all traits/layers & attributes/assets - Object.entries(rarityObject).forEach((entry) => { - const layer = entry[0]; - const assets = entry[1]; - - Object.entries(assets).forEach((asset) => { - // trait/layer related - asset[1].traitOccurance = traitOccurances[layer]; - asset[1].traitOccurancePercentage = - (metadataList.length / asset[1].traitOccurance) * 100; - - // attribute/asset related - asset[1].attributeOccurrencePercentage = - (asset[1].attributeOccurrence / metadataList.length) * 100; - - // TR / SR algorithms specific metadata - if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity - ) { - // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts - // ps: the only difference being that attributeRarityNormed is calculated only once - const totalLayersCnt = Object.keys(traitOccurances).length; - const avgAttributesPerTrait = totalAttributesCnt / totalLayersCnt; - asset[1].attributeFrequency = - asset[1].attributeOccurrence / metadataList.length; - asset[1].traitFrequency = asset[1].traitOccurance > 0 ? 1 : 0; - asset[1].attributeRarity = - metadataList.length / asset[1].attributeOccurrence; - asset[1].attributeRarityNormed = - asset[1].attributeRarity * (avgAttributesPerTrait / totalLayersCnt); - } +const layersDir = `${basePath}/layers`; + +const { layerConfigurations } = require(`${basePath}/src/config.js`); +const { getMetadataItems } = require(`${basePath}/src/metadata.js`); + +const { getElements } = require(`${basePath}/src/main.js`); + +// read json data +let data = getMetadataItems(); +let editionSize = data.length; + +let rarityData = []; + +// intialize layers to chart +console.log("layerconfig", layerConfigurations); +layerConfigurations.forEach((config) => { + let layers = config.layersOrder; + + layers.forEach((layer) => { + // get elements for each layer + let elementsForLayer = []; + let elements = getElements(`${layersDir}/${layer.name}/`); + elements.forEach((element) => { + // just get name and weight for each element + let rarityDataElement = { + trait: element.name, + weight: element.weight.toFixed(0), + occurrence: 0, // initialize at 0 + }; + elementsForLayer.push(rarityDataElement); }); - }); - - return rarityObject; -}; - -// calculates rarity for all items/NFT -const getItemsRarity = (metadataList, rarityObject) => { - switch (network.rarityAlgorithm) { - case RARITY.none: { - return; - } - case RARITY.JaccardDistances: { - return getItemsRarity_JaccardDistances(metadataList); - } - case RARITY.TraitRarity: - case RARITY.StatisticalRarity: - case RARITY.TraitAndStatisticalRarity: { - return getItemsRarity_TSR(metadataList, rarityObject); - } - default: - break; - } -}; - -// calculates rarity for all items/NFT using Jaccard Distances algorithm -const getItemsRarity_JaccardDistances = (metadataList) => { - let z = []; - let avg = []; - - // calculate z(i,j) and avg(i) - for (let i = 0; i < metadataList.length; i++) { - for (let j = 0; j < metadataList.length; j++) { - if (i == j) continue; - - if (z[i] == null) { - z[i] = []; - } - - if (z[i][j] == null || z[j][i] == null) { - const commonTraitsCnt = getObjectCommonCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - const uniqueTraitsCnt = getObjectUniqueCnt( - metadataList[i].attributes, - metadataList[j].attributes - ); - - z[i][j] = commonTraitsCnt / uniqueTraitsCnt; - } + let layerName = + layer.options?.["displayName"] != undefined + ? layer.options?.["displayName"] + : layer.name; + // don't include duplicate layers + if (!rarityData.includes(layer.name)) { + // add elements for each layer to chart + rarityData[layerName] = elementsForLayer; } - - // ps: length-1 because there's always an empty cell in matrix, where i == j - avg[i] = z[i].reduce((a, b) => a + b, 0) / (z[i].length - 1); - } - - // calculate z(i) - let jd = []; - let avgMax = Math.max(...avg); - let avgMin = Math.min(...avg); - - for (let i = 0; i < metadataList.length; i++) { - jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; - } - - const jd_asc = [...jd].sort(function (a, b) { - return a - b; }); - - // add JD rarity data to NFT/item - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity = { - score: jd[i], - }; - if (network.includeRank) { - metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); - } - } - return metadataList; -}; - -// calculates rarity for all items/NFT using Trait/Statistical rarity algorithm(s) -const getItemsRarity_TSR = (metadataList, rarityObject) => { - metadataList.forEach((item) => { - item.rarity = { - usedTraitsCount: item.attributes.length, - }; - // TR specific - if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity - ) { - item.rarity.avgRarity = 0; - item.rarity.rarityScore = 0; - item.rarity.rarityScoreNormed = 0; - } - // SR specific - if ( - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity - ) { - item.rarity.statRarity = 1; - } - - item.attributes.forEach((a) => { - const attributeData = rarityObject[a.trait_type][a.value]; - if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity - ) { - item.rarity.avgRarity += attributeData.attributeFrequency; - item.rarity.rarityScore += attributeData.attributeRarity; - item.rarity.rarityScoreNormed += attributeData.attributeRarityNormed; - } - // SR specific - if ( - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity - ) { - item.rarity.statRarity *= attributeData.attributeFrequency; +}); + +// fill up rarity chart with occurrences from metadata +data.forEach((element) => { + let attributes = element.attributes; + attributes.forEach((attribute) => { + let traitType = attribute.trait_type; + let value = attribute.value; + + let rarityDataTraits = rarityData[traitType]; + rarityDataTraits.forEach((rarityDataTrait) => { + if (rarityDataTrait.trait == value) { + // keep track of occurrences + rarityDataTrait.occurrence++; } }); }); - - // add rarity rank - if (network.includeRank) { - const metadataList_asc = [...metadataList].sort(function (a, b) { - return ( - a.rarity.rarityScore - b.rarity.rarityScore ?? - b.rarity.statRarity - a.rarity.statRarity - ); - }); - for (let i = 0; i < metadataList.length; i++) { - metadataList[i].rarity.rank = - metadataList.length - metadataList_asc.indexOf(metadataList[i]); - } +}); + +// convert occurrences to occurrences string +for (var layer in rarityData) { + for (var attribute in rarityData[layer]) { + // get chance + let chance = ( + (rarityData[layer][attribute].occurrence / editionSize) * + 100 + ).toFixed(2); + + // show two decimal places in percent + rarityData[layer][ + attribute + ].occurrence = `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; } +} - return metadataList; -}; - -module.exports = { getGeneralRarity, getItemsRarity }; +// print out rarity data +for (var layer in rarityData) { + console.log(`Trait type: ${layer}`); + for (var trait in rarityData[layer]) { + console.log(rarityData[layer][trait]); + } + console.log(); +} diff --git a/utils/scripts/rarity.js b/utils/scripts/rarity.js deleted file mode 100644 index bdbf8f111..000000000 --- a/utils/scripts/rarity.js +++ /dev/null @@ -1,89 +0,0 @@ -const basePath = process.cwd(); -const fs = require("fs"); -const { exit } = require("process"); -const { METADATA } = require(`${basePath}/constants/metadata.js`); -const { network } = require(`${basePath}/src/config.js`); -const layersDir = `${basePath}/layers`; - -const { layerConfigurations } = require(`${basePath}/src/config.js`); -const { getMetadataItems } = require(`${basePath}/utils/common.js`); - -const { getElements } = require(`${basePath}/src/main.js`); - -// read json data -let data = getMetadataItems(); -let editionSize = data.length; - -let rarityData = []; - -// intialize layers to chart -console.log("layerconfig", layerConfigurations); -layerConfigurations.forEach((config) => { - let layers = config.layersOrder; - - layers.forEach((layer) => { - // get elements for each layer - let elementsForLayer = []; - let elements = getElements(`${layersDir}/${layer.name}/`); - elements.forEach((element) => { - // just get name and weight for each element - let rarityDataElement = { - trait: element.name, - weight: element.weight.toFixed(0), - occurrence: 0, // initialize at 0 - }; - elementsForLayer.push(rarityDataElement); - }); - let layerName = - layer.options?.["displayName"] != undefined - ? layer.options?.["displayName"] - : layer.name; - // don't include duplicate layers - if (!rarityData.includes(layer.name)) { - // add elements for each layer to chart - rarityData[layerName] = elementsForLayer; - } - }); -}); - -// fill up rarity chart with occurrences from metadata -data.forEach((element) => { - let attributes = element.attributes; - attributes.forEach((attribute) => { - let traitType = attribute.trait_type; - let value = attribute.value; - - let rarityDataTraits = rarityData[traitType]; - rarityDataTraits.forEach((rarityDataTrait) => { - if (rarityDataTrait.trait == value) { - // keep track of occurrences - rarityDataTrait.occurrence++; - } - }); - }); -}); - -// convert occurrences to occurrences string -for (var layer in rarityData) { - for (var attribute in rarityData[layer]) { - // get chance - let chance = ( - (rarityData[layer][attribute].occurrence / editionSize) * - 100 - ).toFixed(2); - - // show two decimal places in percent - rarityData[layer][ - attribute - ].occurrence = `${rarityData[layer][attribute].occurrence} in ${editionSize} editions (${chance} %)`; - } -} - -// print out rarity data -for (var layer in rarityData) { - console.log(`Trait type: ${layer}`); - for (var trait in rarityData[layer]) { - console.log(rarityData[layer][trait]); - } - console.log(); -} diff --git a/utils/scripts/update_info.js b/utils/update_info.js similarity index 96% rename from utils/scripts/update_info.js rename to utils/update_info.js index d058f9303..71bdc36fb 100644 --- a/utils/scripts/update_info.js +++ b/utils/update_info.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { NETWORK } = require(`${basePath}/constants/network.js`); const fs = require("fs"); const { METADATA } = require(`${basePath}/constants/metadata`); -const { getMetadataItems } = require(`${basePath}/utils/common.js`); +const { getMetadataItems } = require(`${basePath}/src/metadata.js`); const { baseUri, From 80cb604125e8d35646f5dae01028981f7418a6d2 Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 26 Apr 2022 01:06:36 +0300 Subject: [PATCH 13/19] move more functions into metadata.js component --- src/config.js | 2 +- src/main.js | 39 ++++++++------------------------------- src/metadata.js | 31 ++++++++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/src/config.js b/src/config.js index adb365d4a..0d67f64aa 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.eth; +const network = NETWORK.egld; // General metadata const namePrefix = "Your Collection"; diff --git a/src/main.js b/src/main.js index 41608519f..9e3540b4b 100644 --- a/src/main.js +++ b/src/main.js @@ -24,7 +24,11 @@ const { solanaMetadata, gif, } = require(`${basePath}/src/config.js`); -const { createMetadataItem } = require(`${basePath}/src/metadata.js`); +const { + createMetadataItem, + writeMetaDataFile, + saveIndividualMetadataFiles, +} = require(`${basePath}/src/metadata.js`); const { getGeneralRarity, getItemsRarity, @@ -267,33 +271,6 @@ const createDna = (_layers) => { return randNum.join(DNA_DELIMITER); }; -const writeMetaData = (_data) => { - fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, - _data - ); -}; - -const saveIndividualMetadataFiles = (abstractedIndexes) => { - let idx = 0; - metadataList.forEach((item) => { - debugLogs - ? console.log( - `Writing metadata for ${ - item.edition || abstractedIndexes[idx] - }: ${JSON.stringify(item)}` - ) - : null; - fs.writeFileSync( - `${buildDir}/${network.jsonDirPrefix}${ - item.edition || abstractedIndexes[idx] - }.json`, - JSON.stringify(item, null, 2) - ); - idx++; - }); -}; - function shuffle(array) { let currentIndex = array.length, randomIndex; @@ -407,7 +384,7 @@ const startCreating = async () => { // calculate rarities (if needed) & save _metadata.json file if (network.metadataType === METADATA.basic) { - writeMetaData(JSON.stringify(metadataList, null, 2)); + writeMetaDataFile(JSON.stringify(metadataList, null, 2)); } else if (network.metadataType === METADATA.rarities) { // calculate rarity for traits/layers & attributes/assets const rarityObject = getGeneralRarity(metadataList); @@ -415,11 +392,11 @@ const startCreating = async () => { // calculate rarity for all items/NFTs metadataList = getItemsRarity(metadataList, rarityObject); } - writeMetaData(JSON.stringify(rarityObject, null, 2)); + writeMetaDataFile(JSON.stringify(rarityObject, null, 2)); } // save individual metadata files - saveIndividualMetadataFiles(abstractedIndexesBackup); + saveIndividualMetadataFiles(metadataList, abstractedIndexesBackup); }; module.exports = { startCreating, buildSetup, getElements }; diff --git a/src/metadata.js b/src/metadata.js index 27b1b5192..f398ee498 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -1,4 +1,5 @@ const basePath = process.cwd(); +const buildDir = `${basePath}/build`; const { NETWORK } = require(`${basePath}/constants/network.js`); const { METADATA } = require(`${basePath}/constants/metadata.js`); const fs = require("fs"); @@ -10,6 +11,7 @@ const { namePrefix, network, solanaMetadata, + debugLogs, } = require(`${basePath}/src/config.js`); // create metadata item with the provided data @@ -93,4 +95,31 @@ const getMetadataItems = () => { } }; -module.exports = { createMetadataItem, getMetadataItems }; +const writeMetaDataFile = (_data) => { + fs.writeFileSync( + `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, + _data + ); +}; + +const saveIndividualMetadataFiles = (metadataList, abstractedIndexes) => { + let idx = 0; + metadataList.forEach((item) => { + debugLogs + ? console.log( + `Writing metadata for ${ + item.edition || abstractedIndexes[idx] + }: ${JSON.stringify(item)}` + ) + : null; + fs.writeFileSync( + `${buildDir}/${network.jsonDirPrefix}${ + item.edition || abstractedIndexes[idx] + }.json`, + JSON.stringify(item, null, 2) + ); + idx++; + }); +}; + +module.exports = { createMetadataItem, getMetadataItems, writeMetaDataFile, saveIndividualMetadataFiles }; From 7725e2c14498f111adafffe9f050ab5db029d30a Mon Sep 17 00:00:00 2001 From: johnykes Date: Tue, 26 Apr 2022 11:04:22 +0300 Subject: [PATCH 14/19] set eth as default --- src/main.js | 6 +++--- src/metadata.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.js b/src/main.js index 9e3540b4b..361e261df 100644 --- a/src/main.js +++ b/src/main.js @@ -26,7 +26,7 @@ const { } = require(`${basePath}/src/config.js`); const { createMetadataItem, - writeMetaDataFile, + writeMetadataFile, saveIndividualMetadataFiles, } = require(`${basePath}/src/metadata.js`); const { @@ -384,7 +384,7 @@ const startCreating = async () => { // calculate rarities (if needed) & save _metadata.json file if (network.metadataType === METADATA.basic) { - writeMetaDataFile(JSON.stringify(metadataList, null, 2)); + writeMetadataFile(JSON.stringify(metadataList, null, 2)); } else if (network.metadataType === METADATA.rarities) { // calculate rarity for traits/layers & attributes/assets const rarityObject = getGeneralRarity(metadataList); @@ -392,7 +392,7 @@ const startCreating = async () => { // calculate rarity for all items/NFTs metadataList = getItemsRarity(metadataList, rarityObject); } - writeMetaDataFile(JSON.stringify(rarityObject, null, 2)); + writeMetadataFile(JSON.stringify(rarityObject, null, 2)); } // save individual metadata files diff --git a/src/metadata.js b/src/metadata.js index f398ee498..0fdddb16a 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -95,7 +95,7 @@ const getMetadataItems = () => { } }; -const writeMetaDataFile = (_data) => { +const writeMetadataFile = (_data) => { fs.writeFileSync( `${buildDir}/${network.jsonDirPrefix}${network.metadataFileName}`, _data @@ -122,4 +122,4 @@ const saveIndividualMetadataFiles = (metadataList, abstractedIndexes) => { }); }; -module.exports = { createMetadataItem, getMetadataItems, writeMetaDataFile, saveIndividualMetadataFiles }; +module.exports = { createMetadataItem, getMetadataItems, writeMetadataFile, saveIndividualMetadataFiles }; From bd25d43fa9e0a8aba6dac19bb84a8dce0a8547b7 Mon Sep 17 00:00:00 2001 From: johnykes Date: Thu, 28 Apr 2022 14:21:32 +0300 Subject: [PATCH 15/19] improve code quality after PR review --- README.md | 14 +++++------ constants/network.js | 8 ++---- constants/rarity.js | 8 +++--- src/config.js | 2 +- src/metadata.js | 58 +++++++++++++++++++++++--------------------- src/rarity.js | 34 +++++++++++++------------- 6 files changed, 62 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index df9bb7cbb..cbb4afbce 100644 --- a/README.md +++ b/README.md @@ -245,16 +245,16 @@ const NETWORK = { startIdx: 1, metadataFileName: "_metadata.json", metadataType: METADATA.rarities, - rarityAlgorithm: RARITY.JaccardDistances, + rarityAlgorithm: RARITY.jaccardDistances, includeRank: true }, ... } ``` -The `metadataType` and `rarityAlgorithm` options can be also found in `network.js` +The `metadataType` and `rarityAlgorithm` options can be also found in `constants/metadata.js` and `constants/rarity.js`. ``` const METADATA = { - // metadata file will contain all individual metadata files (common for eth$, sol$) + // metadata file will contain all individual metadata files (common for eth, sol) // no rarities at all basic: 0, // metadata file will contain only rarity data for traits & attributes (common for egld$) @@ -264,10 +264,10 @@ const METADATA = { const RARITY = { none: 0, - JaccardDistances: 1, // most accurate / recommended - TraitRarity: 2, - StatisticalRarity: 3, - TraitAndStatisticalRarity: 4, + jaccardDistances: 1, // most accurate / recommended + traitRarity: 2, + statisticalRarity: 3, + traitAndstatisticalRarity: 4, }; ``` diff --git a/constants/network.js b/constants/network.js index 46aff9aa0..635af4fe5 100644 --- a/constants/network.js +++ b/constants/network.js @@ -32,13 +32,9 @@ const NETWORK = { ...defaults, name: "egld", metadataType: METADATA.rarities, - rarityAlgorithm: RARITY.JaccardDistances, + rarityAlgorithm: RARITY.jaccardDistances, includeRank: true, }, }; -module.exports = { - NETWORK, - METADATA, - RARITY, -}; +module.exports = { NETWORK }; diff --git a/constants/rarity.js b/constants/rarity.js index a1b4697d8..1477a2c76 100644 --- a/constants/rarity.js +++ b/constants/rarity.js @@ -1,9 +1,9 @@ const RARITY = { none: 0, - JaccardDistances: 1, // most accurate / recommended - TraitRarity: 2, - StatisticalRarity: 3, - TraitAndStatisticalRarity: 4, + jaccardDistances: 1, // most accurate / recommended + traitRarity: 2, + statisticalRarity: 3, + traitAndstatisticalRarity: 4, }; module.exports = { RARITY }; diff --git a/src/config.js b/src/config.js index 0d67f64aa..adb365d4a 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,7 @@ const basePath = process.cwd(); const { MODE } = require(`${basePath}/constants/blend_mode.js`); const { NETWORK } = require(`${basePath}/constants/network.js`); -const network = NETWORK.egld; +const network = NETWORK.eth; // General metadata const namePrefix = "Your Collection"; diff --git a/src/metadata.js b/src/metadata.js index 0fdddb16a..edcd870cb 100644 --- a/src/metadata.js +++ b/src/metadata.js @@ -28,6 +28,7 @@ const createMetadataItem = (attributesList, _dna, _edition) => { rarities: {}, compiler: "HashLips Art Engine", }; + return tempMetadata; } case NETWORK.eth: { tempMetadata = { @@ -41,7 +42,7 @@ const createMetadataItem = (attributesList, _dna, _edition) => { ...extraMetadata, compiler: "HashLips Art Engine", }; - break; + return tempMetadata; } case NETWORK.sol: { tempMetadata = { @@ -65,11 +66,12 @@ const createMetadataItem = (attributesList, _dna, _edition) => { creators: solanaMetadata.creators, }, }; - break; + return tempMetadata; + } + default: { + return tempMetadata; } } - - return tempMetadata; }; // get metadata of all generated NFTs/items @@ -81,18 +83,17 @@ const getMetadataItems = () => { `${basePath}/build/${network.jsonDirPrefix}${network.metadataFileName}` ) ); - } else { - // get metadata from the individual metadata files - const jsonFilePattern = /^\d+.(json)$/i; - const files = fs.readdirSync(`${basePath}/build/${network.jsonDirPrefix}`); - return files - .filter((file) => file.match(jsonFilePattern)) - .map((file) => - JSON.parse( - fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${file}`) - ) - ); } + // get metadata from the individual metadata files + const jsonFilePattern = /^\d+.(json)$/i; + const files = fs.readdirSync(`${basePath}/build/${network.jsonDirPrefix}`); + return files + .filter((file) => file.match(jsonFilePattern)) + .map((file) => + JSON.parse( + fs.readFileSync(`${basePath}/build/${network.jsonDirPrefix}${file}`) + ) + ); }; const writeMetadataFile = (_data) => { @@ -103,23 +104,26 @@ const writeMetadataFile = (_data) => { }; const saveIndividualMetadataFiles = (metadataList, abstractedIndexes) => { - let idx = 0; - metadataList.forEach((item) => { - debugLogs - ? console.log( - `Writing metadata for ${ - item.edition || abstractedIndexes[idx] - }: ${JSON.stringify(item)}` - ) - : null; + metadataList.forEach((item, index) => { + if (debugLogs) { + console.log( + `Writing metadata for ${ + item.edition || abstractedIndexes[index] + }: ${JSON.stringify(item)}` + ); + } fs.writeFileSync( `${buildDir}/${network.jsonDirPrefix}${ - item.edition || abstractedIndexes[idx] + item.edition || abstractedIndexes[index] }.json`, JSON.stringify(item, null, 2) ); - idx++; }); }; -module.exports = { createMetadataItem, getMetadataItems, writeMetadataFile, saveIndividualMetadataFiles }; +module.exports = { + createMetadataItem, + getMetadataItems, + writeMetadataFile, + saveIndividualMetadataFiles, +}; diff --git a/src/rarity.js b/src/rarity.js index 8fdfc339f..343a3ebd6 100644 --- a/src/rarity.js +++ b/src/rarity.js @@ -45,9 +45,9 @@ const getGeneralRarity = (metadataList) => { // TR / SR algorithms specific metadata if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + network.rarityAlgorithm === RARITY.traitRarity || + network.rarityAlgorithm === RARITY.statisticalRarity || + network.rarityAlgorithm === RARITY.traitAndstatisticalRarity ) { // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts // ps: the only difference being that attributeRarityNormed is calculated only once @@ -73,12 +73,12 @@ const getItemsRarity = (metadataList, rarityObject) => { case RARITY.none: { return; } - case RARITY.JaccardDistances: { - return getItemsRarity_JaccardDistances(metadataList); + case RARITY.jaccardDistances: { + return getItemsRarity_jaccardDistances(metadataList); } - case RARITY.TraitRarity: - case RARITY.StatisticalRarity: - case RARITY.TraitAndStatisticalRarity: { + case RARITY.traitRarity: + case RARITY.statisticalRarity: + case RARITY.traitAndstatisticalRarity: { return getItemsRarity_TSR(metadataList, rarityObject); } default: @@ -87,7 +87,7 @@ const getItemsRarity = (metadataList, rarityObject) => { }; // calculates rarity for all items/NFT using Jaccard Distances algorithm -const getItemsRarity_JaccardDistances = (metadataList) => { +const getItemsRarity_jaccardDistances = (metadataList) => { let z = []; let avg = []; @@ -151,8 +151,8 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { }; // TR specific if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + network.rarityAlgorithm === RARITY.traitRarity || + network.rarityAlgorithm === RARITY.traitAndstatisticalRarity ) { item.rarity.avgRarity = 0; item.rarity.rarityScore = 0; @@ -160,8 +160,8 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { } // SR specific if ( - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + network.rarityAlgorithm === RARITY.statisticalRarity || + network.rarityAlgorithm === RARITY.traitAndstatisticalRarity ) { item.rarity.statRarity = 1; } @@ -169,8 +169,8 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { item.attributes.forEach((a) => { const attributeData = rarityObject[a.trait_type][a.value]; if ( - network.rarityAlgorithm === RARITY.TraitRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + network.rarityAlgorithm === RARITY.traitRarity || + network.rarityAlgorithm === RARITY.traitAndstatisticalRarity ) { item.rarity.avgRarity += attributeData.attributeFrequency; item.rarity.rarityScore += attributeData.attributeRarity; @@ -178,8 +178,8 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { } // SR specific if ( - network.rarityAlgorithm === RARITY.StatisticalRarity || - network.rarityAlgorithm === RARITY.TraitAndStatisticalRarity + network.rarityAlgorithm === RARITY.statisticalRarity || + network.rarityAlgorithm === RARITY.traitAndstatisticalRarity ) { item.rarity.statRarity *= attributeData.attributeFrequency; } From a3490d39d1781e5444b0ebebdc22c48c38af61b7 Mon Sep 17 00:00:00 2001 From: johnykes Date: Thu, 28 Apr 2022 14:26:16 +0300 Subject: [PATCH 16/19] fix typo & update README.md --- README.md | 4 ++-- constants/rarity.js | 2 +- src/rarity.js | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index cbb4afbce..76d442639 100644 --- a/README.md +++ b/README.md @@ -251,7 +251,7 @@ const NETWORK = { ... } ``` -The `metadataType` and `rarityAlgorithm` options can be also found in `constants/metadata.js` and `constants/rarity.js`. +The `metadataType` and `rarityAlgorithm` options can be found in `constants/metadata.js` and `constants/rarity.js`. ``` const METADATA = { // metadata file will contain all individual metadata files (common for eth, sol) @@ -267,7 +267,7 @@ const RARITY = { jaccardDistances: 1, // most accurate / recommended traitRarity: 2, statisticalRarity: 3, - traitAndstatisticalRarity: 4, + traitAndStatisticalRarity: 4, }; ``` diff --git a/constants/rarity.js b/constants/rarity.js index 1477a2c76..f76967467 100644 --- a/constants/rarity.js +++ b/constants/rarity.js @@ -3,7 +3,7 @@ const RARITY = { jaccardDistances: 1, // most accurate / recommended traitRarity: 2, statisticalRarity: 3, - traitAndstatisticalRarity: 4, + traitAndStatisticalRarity: 4, }; module.exports = { RARITY }; diff --git a/src/rarity.js b/src/rarity.js index 343a3ebd6..6ba7f8872 100644 --- a/src/rarity.js +++ b/src/rarity.js @@ -47,7 +47,7 @@ const getGeneralRarity = (metadataList) => { if ( network.rarityAlgorithm === RARITY.traitRarity || network.rarityAlgorithm === RARITY.statisticalRarity || - network.rarityAlgorithm === RARITY.traitAndstatisticalRarity + network.rarityAlgorithm === RARITY.traitAndStatisticalRarity ) { // logic from https://github.com/xterr/nft-generator/blob/d8992d2bcfa729a6b2ef443f9404ffa28102111b/src/components/RarityResolver.ts // ps: the only difference being that attributeRarityNormed is calculated only once @@ -78,7 +78,7 @@ const getItemsRarity = (metadataList, rarityObject) => { } case RARITY.traitRarity: case RARITY.statisticalRarity: - case RARITY.traitAndstatisticalRarity: { + case RARITY.traitAndStatisticalRarity: { return getItemsRarity_TSR(metadataList, rarityObject); } default: @@ -152,7 +152,7 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { // TR specific if ( network.rarityAlgorithm === RARITY.traitRarity || - network.rarityAlgorithm === RARITY.traitAndstatisticalRarity + network.rarityAlgorithm === RARITY.traitAndStatisticalRarity ) { item.rarity.avgRarity = 0; item.rarity.rarityScore = 0; @@ -161,7 +161,7 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { // SR specific if ( network.rarityAlgorithm === RARITY.statisticalRarity || - network.rarityAlgorithm === RARITY.traitAndstatisticalRarity + network.rarityAlgorithm === RARITY.traitAndStatisticalRarity ) { item.rarity.statRarity = 1; } @@ -170,7 +170,7 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { const attributeData = rarityObject[a.trait_type][a.value]; if ( network.rarityAlgorithm === RARITY.traitRarity || - network.rarityAlgorithm === RARITY.traitAndstatisticalRarity + network.rarityAlgorithm === RARITY.traitAndStatisticalRarity ) { item.rarity.avgRarity += attributeData.attributeFrequency; item.rarity.rarityScore += attributeData.attributeRarity; @@ -179,7 +179,7 @@ const getItemsRarity_TSR = (metadataList, rarityObject) => { // SR specific if ( network.rarityAlgorithm === RARITY.statisticalRarity || - network.rarityAlgorithm === RARITY.traitAndstatisticalRarity + network.rarityAlgorithm === RARITY.traitAndStatisticalRarity ) { item.rarity.statRarity *= attributeData.attributeFrequency; } From 56ea29d05c69eb7af38afd85ff311146bd524900 Mon Sep 17 00:00:00 2001 From: johnykes Date: Wed, 29 Jun 2022 12:07:42 +0300 Subject: [PATCH 17/19] bug fix --- src/rarity.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rarity.js b/src/rarity.js index 6ba7f8872..efeaf747f 100644 --- a/src/rarity.js +++ b/src/rarity.js @@ -133,11 +133,15 @@ const getItemsRarity_jaccardDistances = (metadataList) => { // add JD rarity data to NFT/item for (let i = 0; i < metadataList.length; i++) { + // remove score in case of duplicates + const jd_i_idx = jd_asc.indexOf(jd[i]); + jd_asc[jd_i_idx] = -1; + metadataList[i].rarity = { score: jd[i], }; if (network.includeRank) { - metadataList[i].rarity.rank = jd.length - jd_asc.indexOf(jd[i]); + metadataList[i].rarity.rank = jd.length - jd_i_idx; } } return metadataList; From 04bfb8294c6caf9efaebd05b548078096f7a3592 Mon Sep 17 00:00:00 2001 From: johnykes Date: Wed, 29 Jun 2022 13:46:55 +0300 Subject: [PATCH 18/19] code improvements (functions instead of strange code) --- src/rarity.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/rarity.js b/src/rarity.js index efeaf747f..8fd3d8242 100644 --- a/src/rarity.js +++ b/src/rarity.js @@ -127,26 +127,35 @@ const getItemsRarity_jaccardDistances = (metadataList) => { jd[i] = ((avg[i] - avgMin) / (avgMax - avgMin)) * 100; } - const jd_asc = [...jd].sort(function (a, b) { + let jd_asc = [...jd].sort(function (a, b) { return a - b; }); // add JD rarity data to NFT/item for (let i = 0; i < metadataList.length; i++) { - // remove score in case of duplicates - const jd_i_idx = jd_asc.indexOf(jd[i]); - jd_asc[jd_i_idx] = -1; + let scoreIndex = getScoreIndex(jd_asc, jd[i]); + jd_asc = markScoreAsUsed(jd_asc, jd[i]); metadataList[i].rarity = { score: jd[i], }; if (network.includeRank) { - metadataList[i].rarity.rank = jd.length - jd_i_idx; + metadataList[i].rarity.rank = jd.length - scoreIndex; } } return metadataList; }; +const getScoreIndex = (jd_asc, score) => { + return jd_asc.indexOf(score); +}; + +const markScoreAsUsed = (jd_asc, score) => { + const jd_i_idx = jd_asc.indexOf(score); + jd_asc[jd_i_idx] = -1; + return jd_asc; +}; + // calculates rarity for all items/NFT using Trait/Statistical rarity algorithm(s) const getItemsRarity_TSR = (metadataList, rarityObject) => { metadataList.forEach((item) => { From 0b818cc5dcf1ba2fe079d2582ab6d1a7742814fe Mon Sep 17 00:00:00 2001 From: johnykes Date: Wed, 29 Jun 2022 13:49:31 +0300 Subject: [PATCH 19/19] code improvements (markScoreAsUsed improvements) --- src/rarity.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/rarity.js b/src/rarity.js index 8fd3d8242..a20045195 100644 --- a/src/rarity.js +++ b/src/rarity.js @@ -134,7 +134,7 @@ const getItemsRarity_jaccardDistances = (metadataList) => { // add JD rarity data to NFT/item for (let i = 0; i < metadataList.length; i++) { let scoreIndex = getScoreIndex(jd_asc, jd[i]); - jd_asc = markScoreAsUsed(jd_asc, jd[i]); + jd_asc = markScoreAsUsed(jd_asc, scoreIndex); metadataList[i].rarity = { score: jd[i], @@ -150,9 +150,8 @@ const getScoreIndex = (jd_asc, score) => { return jd_asc.indexOf(score); }; -const markScoreAsUsed = (jd_asc, score) => { - const jd_i_idx = jd_asc.indexOf(score); - jd_asc[jd_i_idx] = -1; +const markScoreAsUsed = (jd_asc, scoreIndex) => { + jd_asc[scoreIndex] = -1; return jd_asc; };