Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check for duplicate files enhancement #24

Merged
merged 9 commits into from
Aug 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions .coderabbit.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
language: en-US
tone_instructions: ''
early_access: false
enable_free_tier: true
reviews:
profile: chill
request_changes_workflow: false
high_level_summary: true
high_level_summary_placeholder: '@coderabbitai summary'
auto_title_placeholder: '@coderabbitai'
review_status: true
poem: false
collapse_walkthrough: false
sequence_diagrams: true
path_filters: []
path_instructions: []
abort_on_close: true
auto_review:
enabled: true
auto_incremental_review: true
ignore_title_keywords: []
labels: []
drafts: false
base_branches: []
tools:
shellcheck:
enabled: true
ruff:
enabled: true
markdownlint:
enabled: true
github-checks:
enabled: true
timeout_ms: 90000
languagetool:
enabled: true
enabled_only: false
level: default
biome:
enabled: true
hadolint:
enabled: true
swiftlint:
enabled: true
phpstan:
enabled: true
level: default
golangci-lint:
enabled: true
yamllint:
enabled: true
gitleaks:
enabled: true
checkov:
enabled: true
detekt:
enabled: true
eslint:
enabled: true
rubocop:
enabled: true
buf:
enabled: true
chat:
auto_reply: true
knowledge_base:
opt_out: false
learnings:
scope: auto
issues:
scope: auto
jira:
project_keys: []
linear:
team_keys: []
4 changes: 2 additions & 2 deletions demo/esp32/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include <ESPAsyncWebServer.h>
#include "svelteesp32async.h"

#if SVELTEESP32_COUNT != 10
#if SVELTEESP32_COUNT != 11
#error Invalid file count
#endif

Expand Down Expand Up @@ -40,7 +40,7 @@ void loop() {}
#include <PsychicHttp.h>
#include "svelteesp32psychic.h"

#if SVELTEESP32_COUNT != 10
#if SVELTEESP32_COUNT != 11
#error Invalid file count
#endif

Expand Down
Binary file added demo/svelte/dist/favicon.png.gz
Binary file not shown.
Binary file added demo/svelte/dist/gallery/image-1a.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
88 changes: 44 additions & 44 deletions demo/svelte/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions demo/svelte/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
},
"scripts": {
"dev": "vite dev --host",
"build": "vite build",
"build": "vite build && gzip -c ./dist/favicon.png > ./dist/favicon.png.gz && cp ./dist/gallery/image-1.jpg ./dist/gallery/image-1a.jpg",
"preview": "vite preview",
"test": "npm run test:integration && npm run test:unit",
"ts:check": "svelte-check --tsconfig ./tsconfig.json",
Expand All @@ -23,8 +23,8 @@
"@rollup/plugin-swc": "^0.3.1",
"@sveltejs/vite-plugin-svelte": "^3.1.1",
"@tsconfig/svelte": "^5.0.4",
"@typescript-eslint/eslint-plugin": "^8.1.0",
"@typescript-eslint/parser": "^8.1.0",
"@typescript-eslint/eslint-plugin": "^8.2.0",
"@typescript-eslint/parser": "^8.2.0",
"autoprefixer": "^10.4.20",
"eslint": "^9.9.0",
"eslint-config-prettier": "^9.1.0",
Expand Down
6 changes: 3 additions & 3 deletions src/consoleColor.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
export const greenLog = (s: string): string => `\u001B[32m ${s} \u001B[0m`;
export const greenLog = (s: string): string => `\u001B[32m${s}\u001B[0m`;

export const yellowLog = (s: string): string => `\u001B[33m ${s} \u001B[0m`;
export const yellowLog = (s: string): string => `\u001B[33m${s}\u001B[0m`;

export const redLog = (s: string): string => `\u001B[31m ${s} \u001B[0m`;
export const redLog = (s: string): string => `\u001B[31m${s}\u001B[0m`;
35 changes: 28 additions & 7 deletions src/file.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,43 @@
import { createHash } from 'node:crypto';
import { readFileSync } from 'node:fs';
import path from 'node:path';

import { globSync } from 'glob';

import { cmdLine } from './commandLine';
import { redLog } from './consoleColor';
import { redLog, yellowLog } from './consoleColor';

export const getFiles = (): string[] => {
let files = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
files = files.filter((filename) => {
const findSimilarFiles = (files: Map<string, Buffer>): string[][] => {
const contentComparer: Map<string, string[]> = new Map();
for (const [filename, content] of files.entries()) {
const hash = createHash('sha256').update(content).digest('hex');
if (contentComparer.has(hash)) contentComparer.get(hash)!.push(filename);
else contentComparer.set(hash, [filename]);
}

const result: string[][] = [];
for (const filenames of contentComparer.values()) if (filenames.length > 1) result.push(filenames);
return result;
};

export const getFiles = (): Map<string, Buffer> => {
let filenames = globSync('**/*', { cwd: cmdLine.sourcepath, nodir: true });
filenames = filenames.filter((filename) => {
const extension = path.extname(filename);
if (['.gz', '.brottli', '.br'].includes(extension)) {
const original = filename.slice(0, -1 * extension.length);
if (files.includes(original)) {
console.log(redLog(`${filename} skipped because is perhaps a compressed version of ${original}`));
if (filenames.includes(original)) {
console.log(redLog(` ${filename} skipped because is perhaps a compressed version of ${original}`));
return false;
}
}
return true;
});
return files.sort();

const result: Map<string, Buffer> = new Map();
for (const filename of filenames)
result.set(filename, readFileSync(path.join(cmdLine.sourcepath, filename), { flag: 'r' }));
for (const sameFile of findSimilarFiles(result))
console.log(yellowLog(` ${sameFile.join(', ')} files look like identical`));
return result;
};
Loading