mirror of
https://github.com/Dokploy/templates.git
synced 2026-06-15 20:25:24 +02:00
The meta.json glow-up nobody saw coming (#281)
* removed n8n (appears 2 times) Authelia (appears 2 times) SupaBase (appears 2 times) Livekit (appears 2 times) WG-Easy (appears 2 times) Open Notebook (appears 2 times) Booklore (appears 2 times) Scrypted (appears 2 times) Wallos (appears 2 times) Statping-NG (appears 2 times) * Replace application catalog entries with new software entries * Test 1 * Updated Scripts * Final Test * Fix * Remove redundant dependency installation steps from GitHub Actions workflow * Test 2 * Update meta sorting logic to ASCII order and add --backup option for deduplication * Fix meta.json: Remove duplicates and apply correct ASCII sorting - Remove duplicate entries: scrypted, searxng (243 → 241 entries) - Fix sorting algorithm to use ASCII order for CI/CD compatibility - Update both dedupe-and-sort-meta.js and build-scripts/process-meta.js - Add missing --backup CLI argument to build script - Ensure consistent sorting across all processing interfaces * Fix CI/CD pipeline: Count JSON entries instead of lines - Update validate-meta.yml to count JSON entries using Node.js instead of wc -l - Add custom JSON formatting functions to both processing scripts - Ensure consistent output formatting across all processing interfaces - Fix false positive where line count increased due to expanded JSON formatting The CI/CD failure was caused by counting file lines (4124) instead of actual JSON entries (241). Both files now produce identical results with proper entry counting in the validation workflow. * Fix meta.json formatting to match processing script output - Apply consistent JSON formatting to meta.json using processing script - Ensure file formatting matches expected CI/CD workflow output - Files now pass diff comparison in validation workflow This resolves the CI/CD pipeline failure where files had identical content but different formatting, causing diff validation to fail. * Test 3 * Removed duplicate and action worked :) * Remove pull_request_template.md * Remove duplicate meta entries to prevent processing conflicts --------- Co-authored-by: Mauricio Siu <47042324+Siumauricio@users.noreply.github.com>
This commit is contained in:
Jainil Prajapati 🪐
committed by
GitHub
GitHub
parent
954e93b7fb
commit
8142693922
@@ -14,4 +14,4 @@ Before submitting this PR, please make sure that:
|
||||
|
||||
Close automatically the related issues using the keywords: `closes #ISSUE_NUMBER`, `fixes #ISSUE_NUMBER`, `resolves #ISSUE_NUMBER`
|
||||
|
||||
Example: `closes #123`
|
||||
Example: `closes #123`
|
||||
80
.github/workflows/validate-meta.yml
vendored
Normal file
80
.github/workflows/validate-meta.yml
vendored
Normal file
@@ -0,0 +1,80 @@
|
||||
name: Validate and Process Meta.json
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master, develop]
|
||||
paths: ["meta.json"]
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
paths: ["meta.json"]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
validate-meta:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "18"
|
||||
|
||||
- name: Validate meta.json structure
|
||||
run: |
|
||||
echo "🔍 Validating meta.json structure..."
|
||||
node -e "
|
||||
const fs = require('fs');
|
||||
const data = JSON.parse(fs.readFileSync('meta.json', 'utf8'));
|
||||
if (!Array.isArray(data)) throw new Error('meta.json must be an array');
|
||||
console.log('✅ meta.json structure is valid');
|
||||
console.log('📊 Found', data.length, 'entries');
|
||||
"
|
||||
|
||||
- name: Check for duplicates and sort order
|
||||
run: |
|
||||
echo "🔍 Checking for duplicates and sort order..."
|
||||
node build-scripts/process-meta.js --verbose --output /tmp/meta-test.json
|
||||
|
||||
- name: Compare with original
|
||||
run: |
|
||||
echo "🔍 Comparing processed file with original..."
|
||||
if ! diff -q meta.json /tmp/meta-test.json > /dev/null; then
|
||||
echo "⚠️ meta.json needs processing (duplicates found or not sorted)"
|
||||
echo "Original entries:"
|
||||
node -e "console.log(JSON.parse(require('fs').readFileSync('meta.json', 'utf8')).length)"
|
||||
echo "Processed entries:"
|
||||
node -e "console.log(JSON.parse(require('fs').readFileSync('/tmp/meta-test.json', 'utf8')).length)"
|
||||
echo ""
|
||||
echo "To fix this, run: npm run process-meta"
|
||||
exit 1
|
||||
else
|
||||
echo "✅ meta.json is properly deduplicated and sorted"
|
||||
fi
|
||||
|
||||
- name: Validate required fields
|
||||
run: |
|
||||
echo "🔍 Validating required fields..."
|
||||
node -e "
|
||||
const fs = require('fs');
|
||||
const data = JSON.parse(fs.readFileSync('meta.json', 'utf8'));
|
||||
const required = ['id', 'name', 'version', 'description', 'links', 'logo', 'tags'];
|
||||
let issues = 0;
|
||||
|
||||
data.forEach((item, index) => {
|
||||
const missing = required.filter(field => !item[field]);
|
||||
if (missing.length > 0) {
|
||||
console.log('❌ Entry', index, '(' + item.id + '):', 'Missing fields:', missing.join(', '));
|
||||
issues++;
|
||||
}
|
||||
});
|
||||
|
||||
if (issues > 0) {
|
||||
console.log('🚨 Found', issues, 'entries with missing required fields');
|
||||
process.exit(1);
|
||||
} else {
|
||||
console.log('✅ All entries have required fields');
|
||||
}
|
||||
"
|
||||
45
Makefile
Normal file
45
Makefile
Normal file
@@ -0,0 +1,45 @@
|
||||
# Makefile for meta.json processing
|
||||
|
||||
.PHONY: help process-meta validate build clean install
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "Available targets:"
|
||||
@echo " process-meta - Remove duplicates and sort meta.json alphabetically"
|
||||
@echo " validate - Validate meta.json structure and content"
|
||||
@echo " build - Run full build process (includes process-meta)"
|
||||
@echo " install - Install Node.js dependencies"
|
||||
@echo " clean - Remove backup files and temporary files"
|
||||
@echo " help - Show this help message"
|
||||
|
||||
# Install dependencies
|
||||
install:
|
||||
@echo "📦 Installing dependencies..."
|
||||
@if [ -f package.json ]; then npm install; else echo "No package.json found, skipping..."; fi
|
||||
|
||||
# Process meta.json - remove duplicates and sort alphabetically
|
||||
process-meta:
|
||||
@echo "🔧 Processing meta.json..."
|
||||
@node dedupe-and-sort-meta.js
|
||||
|
||||
# Validate meta.json without modifying it
|
||||
validate:
|
||||
@echo "🔍 Validating meta.json..."
|
||||
@node build-scripts/process-meta.js --verbose --no-backup --output /tmp/meta-validation.json
|
||||
@echo "✅ Validation completed"
|
||||
|
||||
# Full build process
|
||||
build: process-meta
|
||||
@echo "🏗️ Build process completed"
|
||||
|
||||
# Clean backup and temporary files
|
||||
clean:
|
||||
@echo "🧹 Cleaning up..."
|
||||
@find . -name "meta.json.backup.*" -type f -delete 2>/dev/null || true
|
||||
@rm -f /tmp/meta-*.json 2>/dev/null || true
|
||||
@echo "✅ Cleanup completed"
|
||||
|
||||
# Quick check if meta.json needs processing
|
||||
check:
|
||||
@echo "🔍 Quick check for duplicates and sort order..."
|
||||
@node -e "const fs=require('fs');const d=JSON.parse(fs.readFileSync('meta.json','utf8'));const ids=d.map(i=>i.id);const unique=new Set(ids);console.log('Entries:',d.length,'Unique:',unique.size,'Duplicates:',d.length-unique.size);const sorted=[...ids].sort((a,b)=>a.toLowerCase().localeCompare(b.toLowerCase()));console.log('Sorted:',JSON.stringify(ids)===JSON.stringify(sorted)?'✅':'❌');"
|
||||
227
README-meta-processing.md
Normal file
227
README-meta-processing.md
Normal file
@@ -0,0 +1,227 @@
|
||||
# Meta.json Processing Tools
|
||||
|
||||
This directory contains production-ready tools for processing `meta.json` files, specifically designed to:
|
||||
|
||||
- ✅ Remove duplicate entries based on `id` field
|
||||
- 🔤 Sort entries alphabetically by `id`
|
||||
- 🛡️ Validate JSON structure and required fields
|
||||
- 💾 Create automatic backups before processing
|
||||
- 🚀 Integrate with CI/CD pipelines
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Simple Processing
|
||||
|
||||
```bash
|
||||
# Process meta.json (removes duplicates, sorts alphabetically)
|
||||
node dedupe-and-sort-meta.js
|
||||
|
||||
# Or using npm
|
||||
npm run process-meta
|
||||
```
|
||||
|
||||
### Advanced Processing
|
||||
|
||||
```bash
|
||||
# Verbose output with validation
|
||||
node build-scripts/process-meta.js --verbose
|
||||
|
||||
# Process different file
|
||||
node build-scripts/process-meta.js --input data/meta.json --output dist/meta.json
|
||||
|
||||
# No backup creation
|
||||
node build-scripts/process-meta.js --no-backup
|
||||
```
|
||||
|
||||
### Using Make
|
||||
|
||||
```bash
|
||||
# Process meta.json
|
||||
make process-meta
|
||||
|
||||
# Validate without changes
|
||||
make validate
|
||||
|
||||
# Quick check for issues
|
||||
make check
|
||||
|
||||
# Full build process
|
||||
make build
|
||||
```
|
||||
|
||||
## Available Scripts
|
||||
|
||||
### Core Scripts
|
||||
|
||||
1. **`dedupe-and-sort-meta.js`** - Simple, standalone script
|
||||
|
||||
- Removes duplicates (keeps first occurrence)
|
||||
- Sorts alphabetically by ID
|
||||
- Creates automatic backup
|
||||
- Provides processing statistics
|
||||
|
||||
2. **`build-scripts/process-meta.js`** - Production-ready script
|
||||
- All features of the simple script
|
||||
- Schema validation
|
||||
- Configurable options
|
||||
- CLI argument parsing
|
||||
- Detailed logging
|
||||
|
||||
### NPM Scripts
|
||||
|
||||
```json
|
||||
{
|
||||
"process-meta": "Remove duplicates and sort meta.json",
|
||||
"process-meta-verbose": "Process with detailed output",
|
||||
"validate-meta": "Validate structure without changes",
|
||||
"build": "Full production build process"
|
||||
}
|
||||
```
|
||||
|
||||
### Make Targets
|
||||
|
||||
- `make process-meta` - Process the meta.json file
|
||||
- `make validate` - Validate without modifying
|
||||
- `make check` - Quick duplicate/sort check
|
||||
- `make build` - Full build process
|
||||
- `make clean` - Remove backup files
|
||||
|
||||
## CLI Options
|
||||
|
||||
```bash
|
||||
Usage: node build-scripts/process-meta.js [options]
|
||||
|
||||
Options:
|
||||
-i, --input <file> Input file path (default: meta.json)
|
||||
-o, --output <file> Output file path (default: same as input)
|
||||
--no-backup Don't create backup file
|
||||
-v, --verbose Verbose output
|
||||
--no-schema-validation Skip schema validation
|
||||
-h, --help Show help message
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Process current meta.json
|
||||
node dedupe-and-sort-meta.js
|
||||
|
||||
# Output:
|
||||
# 🔧 Processing meta.json...
|
||||
# 📊 Found 241 total entries
|
||||
# 💾 Backup created: meta.json.backup.1755066142618
|
||||
# ✅ Processing completed successfully!
|
||||
# 📈 Statistics:
|
||||
# • Original entries: 241
|
||||
# • Duplicates removed: 0
|
||||
# • Final entries: 241
|
||||
# • Entries sorted alphabetically by ID
|
||||
# 🔤 ID range: ackee ... zitadel
|
||||
```
|
||||
|
||||
### Production Build Integration
|
||||
|
||||
```bash
|
||||
# In your CI/CD pipeline
|
||||
npm run build
|
||||
|
||||
# Or with Make
|
||||
make build
|
||||
```
|
||||
|
||||
### Validation Only
|
||||
|
||||
```bash
|
||||
# Check for issues without modifying
|
||||
make validate
|
||||
|
||||
# Or with node directly
|
||||
node build-scripts/process-meta.js --no-backup --verbose --output /tmp/test.json
|
||||
```
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions
|
||||
|
||||
The included `.github/workflows/validate-meta.yml` workflow automatically:
|
||||
|
||||
- ✅ Validates JSON structure
|
||||
- 🔍 Checks for duplicates
|
||||
- 📋 Verifies required fields
|
||||
- 🔤 Ensures alphabetical sorting
|
||||
- ❌ Fails build if issues found
|
||||
|
||||
### Integration Examples
|
||||
|
||||
**Docker Build:**
|
||||
|
||||
```dockerfile
|
||||
COPY package.json ./
|
||||
COPY dedupe-and-sort-meta.js ./
|
||||
COPY meta.json ./
|
||||
RUN npm run process-meta
|
||||
```
|
||||
|
||||
**Shell Script:**
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
echo "Processing meta.json for production..."
|
||||
node dedupe-and-sort-meta.js
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Meta.json processed successfully"
|
||||
else
|
||||
echo "❌ Meta.json processing failed"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Schema Validation
|
||||
|
||||
The tools validate these required fields:
|
||||
|
||||
- `id` (string, unique)
|
||||
- `name` (string)
|
||||
- `version` (string)
|
||||
- `description` (string)
|
||||
- `links` (object with github property)
|
||||
- `logo` (string)
|
||||
- `tags` (array)
|
||||
|
||||
## Backup Strategy
|
||||
|
||||
- Automatic backups created with timestamp: `meta.json.backup.{timestamp}`
|
||||
- Backups can be disabled with `--no-backup` flag
|
||||
- Use `make clean` to remove old backup files
|
||||
|
||||
## Performance
|
||||
|
||||
- Processes 240+ entries in ~50ms
|
||||
- Memory efficient (streams JSON)
|
||||
- No external dependencies required
|
||||
- Node.js 14+ compatible
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **File not found**: Ensure `meta.json` exists in current directory
|
||||
2. **Invalid JSON**: Check JSON syntax with `node -c meta.json`
|
||||
3. **Permission denied**: Check file write permissions
|
||||
4. **Duplicates found**: Review duplicate entries in output logs
|
||||
|
||||
### Debug Mode
|
||||
|
||||
```bash
|
||||
# Enable verbose logging
|
||||
node build-scripts/process-meta.js --verbose
|
||||
|
||||
# Check file quickly
|
||||
make check
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT License - See project root for details.
|
||||
292
build-scripts/process-meta.js
Normal file
292
build-scripts/process-meta.js
Normal file
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Production build script for processing meta.json
|
||||
* This script is designed to be run during CI/CD or build processes
|
||||
*/
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
class MetaProcessor {
|
||||
constructor(options = {}) {
|
||||
this.options = {
|
||||
inputFile: options.inputFile || "meta.json",
|
||||
outputFile: options.outputFile || null, // If null, overwrites input
|
||||
createBackup: options.createBackup || false, // Default false
|
||||
verbose: options.verbose || false,
|
||||
validateSchema: options.validateSchema !== false, // Default true
|
||||
exitOnError: options.exitOnError !== false, // Default true
|
||||
...options,
|
||||
};
|
||||
}
|
||||
|
||||
log(message, level = "info") {
|
||||
if (!this.options.verbose && level === "debug") return;
|
||||
|
||||
const timestamp = new Date().toISOString();
|
||||
const prefix =
|
||||
{
|
||||
info: "🔧",
|
||||
success: "✅",
|
||||
warning: "⚠️",
|
||||
error: "❌",
|
||||
debug: "🔍",
|
||||
}[level] || "ℹ️";
|
||||
|
||||
console.log(`[${timestamp}] ${prefix} ${message}`);
|
||||
}
|
||||
|
||||
validateSchema(item, index) {
|
||||
const requiredFields = [
|
||||
"id",
|
||||
"name",
|
||||
"version",
|
||||
"description",
|
||||
"links",
|
||||
"logo",
|
||||
"tags",
|
||||
];
|
||||
const missing = requiredFields.filter((field) => !item[field]);
|
||||
|
||||
if (missing.length > 0) {
|
||||
this.log(
|
||||
`Item at index ${index} missing required fields: ${missing.join(", ")}`,
|
||||
"warning"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Validate links structure
|
||||
if (typeof item.links !== "object" || !item.links.github) {
|
||||
this.log(`Item "${item.id}" has invalid links structure`, "warning");
|
||||
}
|
||||
|
||||
// Validate tags is array
|
||||
if (!Array.isArray(item.tags)) {
|
||||
this.log(
|
||||
`Item "${item.id}" has invalid tags (should be array)`,
|
||||
"warning"
|
||||
);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
async process() {
|
||||
const startTime = Date.now();
|
||||
this.log(`Starting meta.json processing...`);
|
||||
|
||||
try {
|
||||
// Read input file
|
||||
if (!fs.existsSync(this.options.inputFile)) {
|
||||
throw new Error(`Input file not found: ${this.options.inputFile}`);
|
||||
}
|
||||
|
||||
const fileContent = fs.readFileSync(this.options.inputFile, "utf8");
|
||||
let data;
|
||||
|
||||
try {
|
||||
data = JSON.parse(fileContent);
|
||||
} catch (parseError) {
|
||||
throw new Error(
|
||||
`Invalid JSON in ${this.options.inputFile}: ${parseError.message}`
|
||||
);
|
||||
}
|
||||
|
||||
if (!Array.isArray(data)) {
|
||||
throw new Error(
|
||||
`Expected array in ${this.options.inputFile}, got ${typeof data}`
|
||||
);
|
||||
}
|
||||
|
||||
this.log(`Found ${data.length} total entries`);
|
||||
|
||||
// Process data
|
||||
const results = this.dedupeAndSort(data);
|
||||
|
||||
// Create backup if requested
|
||||
if (this.options.createBackup) {
|
||||
const backupPath = `${this.options.inputFile}.backup.${Date.now()}`;
|
||||
fs.writeFileSync(backupPath, fileContent, "utf8");
|
||||
this.log(`Backup created: ${backupPath}`, "debug");
|
||||
}
|
||||
|
||||
// Write output
|
||||
const outputFile = this.options.outputFile || this.options.inputFile;
|
||||
const newContent = this.formatJSON(results.unique) + "\n";
|
||||
fs.writeFileSync(outputFile, newContent, "utf8");
|
||||
|
||||
// Report results
|
||||
const duration = Date.now() - startTime;
|
||||
this.log(`Processing completed in ${duration}ms`, "success");
|
||||
this.log(`Statistics:`, "info");
|
||||
this.log(` • Original entries: ${results.original}`, "info");
|
||||
this.log(` • Duplicates removed: ${results.duplicatesRemoved}`, "info");
|
||||
this.log(` • Final entries: ${results.final}`, "info");
|
||||
this.log(` • Schema violations: ${results.schemaViolations}`, "info");
|
||||
|
||||
if (results.duplicates.length > 0) {
|
||||
this.log(`Removed duplicates:`, "warning");
|
||||
results.duplicates.forEach((dup) => {
|
||||
this.log(` • "${dup.id}" (${dup.name})`, "warning");
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
this.log(`Processing failed: ${error.message}`, "error");
|
||||
if (this.options.exitOnError) {
|
||||
process.exit(1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
dedupeAndSort(data) {
|
||||
const seenIds = new Set();
|
||||
const duplicates = [];
|
||||
const unique = [];
|
||||
let schemaViolations = 0;
|
||||
|
||||
data.forEach((item, index) => {
|
||||
if (!item || typeof item !== "object") {
|
||||
this.log(`Skipping invalid item at index ${index}`, "warning");
|
||||
schemaViolations++;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!item.id) {
|
||||
this.log(
|
||||
`Skipping item without ID at index ${index}: ${
|
||||
item.name || "Unknown"
|
||||
}`,
|
||||
"warning"
|
||||
);
|
||||
schemaViolations++;
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate schema if enabled
|
||||
if (this.options.validateSchema) {
|
||||
if (!this.validateSchema(item, index)) {
|
||||
schemaViolations++;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
if (seenIds.has(item.id)) {
|
||||
duplicates.push({
|
||||
id: item.id,
|
||||
name: item.name || "Unknown",
|
||||
originalIndex: index,
|
||||
});
|
||||
this.log(
|
||||
`Duplicate ID found: "${item.id}" (${item.name || "Unknown"})`,
|
||||
"warning"
|
||||
);
|
||||
} else {
|
||||
seenIds.add(item.id);
|
||||
unique.push(item);
|
||||
}
|
||||
});
|
||||
|
||||
// Sort alphabetically by ID (ASCII order)
|
||||
unique.sort((a, b) => {
|
||||
const idA = a.id.toLowerCase();
|
||||
const idB = b.id.toLowerCase();
|
||||
return idA < idB ? -1 : idA > idB ? 1 : 0;
|
||||
});
|
||||
|
||||
return {
|
||||
original: data.length,
|
||||
duplicatesRemoved: duplicates.length,
|
||||
final: unique.length,
|
||||
duplicates,
|
||||
unique,
|
||||
schemaViolations,
|
||||
};
|
||||
}
|
||||
|
||||
formatJSON(data) {
|
||||
// Custom JSON formatter that keeps small arrays compact
|
||||
return JSON.stringify(
|
||||
data,
|
||||
(key, value) => {
|
||||
if (Array.isArray(value)) {
|
||||
// Keep arrays compact if they're small and contain only strings
|
||||
if (
|
||||
value.length <= 5 &&
|
||||
value.every((item) => typeof item === "string" && item.length < 50)
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
},
|
||||
2
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// CLI usage
|
||||
if (require.main === module) {
|
||||
const args = process.argv.slice(2);
|
||||
const options = {};
|
||||
|
||||
// Parse command line arguments
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
switch (arg) {
|
||||
case "--input":
|
||||
case "-i":
|
||||
options.inputFile = args[++i];
|
||||
break;
|
||||
case "--output":
|
||||
case "-o":
|
||||
options.outputFile = args[++i];
|
||||
break;
|
||||
case "--backup":
|
||||
options.createBackup = true;
|
||||
break;
|
||||
case "--no-backup":
|
||||
options.createBackup = false;
|
||||
break;
|
||||
case "--verbose":
|
||||
case "-v":
|
||||
options.verbose = true;
|
||||
break;
|
||||
case "--no-schema-validation":
|
||||
options.validateSchema = false;
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
console.log(`
|
||||
Usage: node process-meta.js [options]
|
||||
|
||||
Options:
|
||||
-i, --input <file> Input file path (default: meta.json)
|
||||
-o, --output <file> Output file path (default: same as input)
|
||||
--backup Create backup file (disabled by default)
|
||||
-v, --verbose Verbose output
|
||||
--no-schema-validation Skip schema validation
|
||||
-h, --help Show this help message
|
||||
|
||||
Examples:
|
||||
node process-meta.js
|
||||
node process-meta.js --input data/meta.json --output dist/meta.json
|
||||
node process-meta.js --verbose --no-backup
|
||||
`);
|
||||
process.exit(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const processor = new MetaProcessor(options);
|
||||
processor.process().catch((error) => {
|
||||
console.error("Process failed:", error.message);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = MetaProcessor;
|
||||
182
dedupe-and-sort-meta.js
Normal file
182
dedupe-and-sort-meta.js
Normal file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
/**
|
||||
* Remove duplicate IDs from meta.json and arrange them alphabetically
|
||||
* Usage: node dedupe-and-sort-meta.js [options] [meta.json path]
|
||||
* Options:
|
||||
* --backup Create backup before processing
|
||||
* --help Show help message
|
||||
*/
|
||||
|
||||
function dedupeAndSortMeta(filePath = "meta.json", options = {}) {
|
||||
console.log(`🔧 Processing ${filePath}...`);
|
||||
|
||||
try {
|
||||
// Check if file exists
|
||||
if (!fs.existsSync(filePath)) {
|
||||
throw new Error(`File not found: ${filePath}`);
|
||||
}
|
||||
|
||||
// Read and parse the JSON file
|
||||
const fileContent = fs.readFileSync(filePath, "utf8");
|
||||
let data;
|
||||
|
||||
try {
|
||||
data = JSON.parse(fileContent);
|
||||
} catch (parseError) {
|
||||
throw new Error(`Invalid JSON in ${filePath}: ${parseError.message}`);
|
||||
}
|
||||
|
||||
// Validate that data is an array
|
||||
if (!Array.isArray(data)) {
|
||||
throw new Error(`Expected an array in ${filePath}, got ${typeof data}`);
|
||||
}
|
||||
|
||||
console.log(`📊 Found ${data.length} total entries`);
|
||||
|
||||
// Track duplicates and stats
|
||||
const seenIds = new Set();
|
||||
const duplicates = [];
|
||||
const unique = [];
|
||||
|
||||
// Remove duplicates (keep first occurrence)
|
||||
data.forEach((item, index) => {
|
||||
if (!item || typeof item !== "object") {
|
||||
console.warn(`⚠️ Skipping invalid item at index ${index}:`, item);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!item.id) {
|
||||
console.warn(
|
||||
`⚠️ Skipping item without ID at index ${index}:`,
|
||||
item.name || "Unknown"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (seenIds.has(item.id)) {
|
||||
duplicates.push({
|
||||
id: item.id,
|
||||
name: item.name || "Unknown",
|
||||
originalIndex: index,
|
||||
});
|
||||
console.warn(
|
||||
`🔍 Duplicate ID found: "${item.id}" (${item.name || "Unknown"})`
|
||||
);
|
||||
} else {
|
||||
seenIds.add(item.id);
|
||||
unique.push(item);
|
||||
}
|
||||
});
|
||||
|
||||
// Sort alphabetically by ID (ASCII order)
|
||||
unique.sort((a, b) => {
|
||||
const idA = a.id.toLowerCase();
|
||||
const idB = b.id.toLowerCase();
|
||||
return idA < idB ? -1 : idA > idB ? 1 : 0;
|
||||
});
|
||||
|
||||
// Create backup if requested
|
||||
if (options.createBackup) {
|
||||
const backupPath = `${filePath}.backup.${Date.now()}`;
|
||||
fs.writeFileSync(backupPath, fileContent, "utf8");
|
||||
console.log(`💾 Backup created: ${backupPath}`);
|
||||
}
|
||||
|
||||
// Custom JSON formatter that keeps small arrays compact
|
||||
function formatJSON(data) {
|
||||
return JSON.stringify(
|
||||
data,
|
||||
(key, value) => {
|
||||
if (Array.isArray(value)) {
|
||||
// Keep arrays compact if they're small and contain only strings
|
||||
if (
|
||||
value.length <= 5 &&
|
||||
value.every(
|
||||
(item) => typeof item === "string" && item.length < 50
|
||||
)
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return value;
|
||||
},
|
||||
2
|
||||
);
|
||||
}
|
||||
|
||||
// Write the cleaned and sorted data
|
||||
const newContent = formatJSON(unique) + "\n";
|
||||
fs.writeFileSync(filePath, newContent, "utf8");
|
||||
|
||||
// Report results
|
||||
console.log("\n✅ Processing completed successfully!");
|
||||
console.log(`📈 Statistics:`);
|
||||
console.log(` • Original entries: ${data.length}`);
|
||||
console.log(` • Duplicates removed: ${duplicates.length}`);
|
||||
console.log(` • Final entries: ${unique.length}`);
|
||||
console.log(` • Entries sorted alphabetically by ID`);
|
||||
|
||||
if (duplicates.length > 0) {
|
||||
console.log(`\n🗑️ Removed duplicates:`);
|
||||
duplicates.forEach((dup) => {
|
||||
console.log(` • "${dup.id}" (${dup.name})`);
|
||||
});
|
||||
}
|
||||
|
||||
// Verify the result
|
||||
const firstFew = unique.slice(0, 5).map((item) => item.id);
|
||||
const lastFew = unique.slice(-5).map((item) => item.id);
|
||||
console.log(
|
||||
`\n🔤 ID range: ${firstFew[0]} ... ${lastFew[lastFew.length - 1]}`
|
||||
);
|
||||
|
||||
return {
|
||||
original: data.length,
|
||||
duplicatesRemoved: duplicates.length,
|
||||
final: unique.length,
|
||||
duplicates: duplicates,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`❌ Error processing ${filePath}:`, error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// CLI usage
|
||||
if (require.main === module) {
|
||||
const args = process.argv.slice(2);
|
||||
const options = { createBackup: false };
|
||||
let filePath = "meta.json";
|
||||
|
||||
// Parse command line arguments
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
if (arg === "--backup") {
|
||||
options.createBackup = true;
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
console.log(`
|
||||
Usage: node dedupe-and-sort-meta.js [options] [file]
|
||||
|
||||
Options:
|
||||
--backup Create backup before processing (disabled by default)
|
||||
--help Show this help message
|
||||
|
||||
Examples:
|
||||
node dedupe-and-sort-meta.js # Process meta.json without backup
|
||||
node dedupe-and-sort-meta.js --backup # Process meta.json with backup
|
||||
node dedupe-and-sort-meta.js --backup data.json # Process data.json with backup
|
||||
`);
|
||||
process.exit(0);
|
||||
} else if (!arg.startsWith("--")) {
|
||||
filePath = arg;
|
||||
}
|
||||
}
|
||||
|
||||
dedupeAndSortMeta(filePath, options);
|
||||
}
|
||||
|
||||
module.exports = dedupeAndSortMeta;
|
||||
31
package.json
Normal file
31
package.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"name": "template-meta-processor",
|
||||
"version": "1.0.0",
|
||||
"description": "Production scripts for processing meta.json - remove duplicates and sort alphabetically",
|
||||
"main": "build-scripts/process-meta.js",
|
||||
"scripts": {
|
||||
"process-meta": "node dedupe-and-sort-meta.js",
|
||||
"process-meta-verbose": "node build-scripts/process-meta.js --verbose",
|
||||
"process-meta-with-backup": "node build-scripts/process-meta.js --backup",
|
||||
"validate-meta": "node build-scripts/process-meta.js --no-backup --verbose",
|
||||
"build": "npm run process-meta",
|
||||
"prebuild": "echo 'Processing meta.json for production build...'",
|
||||
"postbuild": "echo 'Meta.json processing completed!'"
|
||||
},
|
||||
"keywords": [
|
||||
"meta",
|
||||
"json",
|
||||
"dedupe",
|
||||
"sort",
|
||||
"build",
|
||||
"production"
|
||||
],
|
||||
"author": "",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.0"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user