commit 126cd6931feee535a99acce5b847387c5d346d81 · nekomimi.pet/wisp.place-monorepo

-1

.tangled/workflows/test.yml

···

       17
       17
        
       

     

       18
       18
        
             # have to regenerate otherwise it wont install necessary dependencies to run

     

       19
       19
        
             rm -rf bun.lock package-lock.json

     

       20
       20
       -
             bun install @oven/bun-linux-aarch64

     

       21
       20
        
             bun install

     

       22
       21
        
       

     

       23
       22
        
         - name: run all tests

+74

.wispignore.json

···

       1
       1
       +
       {

     

       2
       2
       +
         "version": "1.0.0",

     

       3
       3
       +
         "description": "Default ignore patterns for wisp.place uploads",

     

       4
       4
       +
         "patterns": [

     

       5
       5
       +
           ".git",

     

       6
       6
       +
           ".git/**",

     

       7
       7
       +
           ".github",

     

       8
       8
       +
           ".github/**",

     

       9
       9
       +
           ".gitlab",

     

       10
       10
       +
           ".gitlab/**",

     

       11
       11
       +
           ".DS_Store",

     

       12
       12
       +
           ".wisp.metadata.json",

     

       13
       13
       +
           ".env",

     

       14
       14
       +
           ".env.*",

     

       15
       15
       +
           "node_modules",

     

       16
       16
       +
           "node_modules/**",

     

       17
       17
       +
           "Thumbs.db",

     

       18
       18
       +
           "desktop.ini",

     

       19
       19
       +
           "._*",

     

       20
       20
       +
           ".Spotlight-V100",

     

       21
       21
       +
           ".Spotlight-V100/**",

     

       22
       22
       +
           ".Trashes",

     

       23
       23
       +
           ".Trashes/**",

     

       24
       24
       +
           ".fseventsd",

     

       25
       25
       +
           ".fseventsd/**",

     

       26
       26
       +
           ".cache",

     

       27
       27
       +
           ".cache/**",

     

       28
       28
       +
           ".temp",

     

       29
       29
       +
           ".temp/**",

     

       30
       30
       +
           ".tmp",

     

       31
       31
       +
           ".tmp/**",

     

       32
       32
       +
           "__pycache__",

     

       33
       33
       +
           "__pycache__/**",

     

       34
       34
       +
           "*.pyc",

     

       35
       35
       +
           ".venv",

     

       36
       36
       +
           ".venv/**",

     

       37
       37
       +
           "venv",

     

       38
       38
       +
           "venv/**",

     

       39
       39
       +
           "env",

     

       40
       40
       +
           "env/**",

     

       41
       41
       +
           "*.swp",

     

       42
       42
       +
           "*.swo",

     

       43
       43
       +
           "*~",

     

       44
       44
       +
           ".tangled",

     

       45
       45
       +
           ".tangled/**"

     

       46
       46
       +
         ],

     

       47
       47
       +
         "comments": {

     

       48
       48
       +
           ".git": "Version control - thousands of files",

     

       49
       49
       +
           ".github": "GitHub workflows and configuration",

     

       50
       50
       +
           ".gitlab": "GitLab CI/CD configuration",

     

       51
       51
       +
           ".DS_Store": "macOS metadata - can leak info",

     

       52
       52
       +
           ".wisp.metadata.json": "Wisp internal metadata",

     

       53
       53
       +
           ".env": "Environment variables with secrets",

     

       54
       54
       +
           "node_modules": "Dependency folder - can be 100,000+ files",

     

       55
       55
       +
           "Thumbs.db": "Windows thumbnail cache",

     

       56
       56
       +
           "desktop.ini": "Windows folder config",

     

       57
       57
       +
           "._*": "macOS resource fork files",

     

       58
       58
       +
           ".Spotlight-V100": "macOS Spotlight index",

     

       59
       59
       +
           ".Trashes": "macOS trash folder",

     

       60
       60
       +
           ".fseventsd": "macOS filesystem events",

     

       61
       61
       +
           ".cache": "Cache directories",

     

       62
       62
       +
           ".temp": "Temporary directories",

     

       63
       63
       +
           ".tmp": "Temporary directories",

     

       64
       64
       +
           "__pycache__": "Python cache",

     

       65
       65
       +
           "*.pyc": "Python compiled files",

     

       66
       66
       +
           ".venv": "Python virtual environments",

     

       67
       67
       +
           "venv": "Python virtual environments",

     

       68
       68
       +
           "env": "Python virtual environments",

     

       69
       69
       +
           "*.swp": "Vim swap files",

     

       70
       70
       +
           "*.swo": "Vim swap files",

     

       71
       71
       +
           "*~": "Editor backup files",

     

       72
       72
       +
           ".tangled": "Tangled directory"

     

       73
       73
       +
         }

     

       74
       74
       +
       }

+9 -8

apps/main-app/package.json

···

       10
       10
        
           "screenshot": "bun run scripts/screenshot-sites.ts"

     

       11
       11
        
         },

     

       12
       12
        
         "dependencies": {

     

       13
       13
       -
           "@wisp/lexicons": "workspace:*",

     

       14
       14
       -
           "@wisp/constants": "workspace:*",

     

       15
       15
       -
           "@wisp/observability": "workspace:*",

     

       16
       16
       -
           "@wisp/atproto-utils": "workspace:*",

     

       17
       17
       -
           "@wisp/database": "workspace:*",

     

       18
       18
       -
           "@wisp/fs-utils": "workspace:*",

     

       19
       13
        
           "@atproto/api": "^0.17.3",

     

       20
       14
        
           "@atproto/common-web": "^0.4.5",

     

       21
       15
        
           "@atproto/jwk-jose": "^0.1.11",

     
···

       34
       28
        
           "@radix-ui/react-slot": "^1.2.3",

     

       35
       29
        
           "@radix-ui/react-tabs": "^1.1.13",

     

       36
       30
        
           "@tanstack/react-query": "^5.90.2",

     

       31
       31
       +
           "@wisp/atproto-utils": "workspace:*",

     

       32
       32
       +
           "@wisp/constants": "workspace:*",

     

       33
       33
       +
           "@wisp/database": "workspace:*",

     

       34
       34
       +
           "@wisp/fs-utils": "workspace:*",

     

       35
       35
       +
           "@wisp/lexicons": "workspace:*",

     

       36
       36
       +
           "@wisp/observability": "workspace:*",

     

       37
       37
        
           "actor-typeahead": "^0.1.1",

     

       38
       38
        
           "atproto-ui": "^0.11.3",

     

       39
       39
       +
           "bun-plugin-tailwind": "^0.1.2",

     

       39
       40
        
           "class-variance-authority": "^0.7.1",

     

       40
       41
        
           "clsx": "^2.1.1",

     

       41
       42
        
           "elysia": "latest",

     

       43
       43
       +
           "ignore": "^7.0.5",

     

       42
       44
        
           "iron-session": "^8.0.4",

     

       43
       45
        
           "lucide-react": "^0.546.0",

     

       44
       46
        
           "multiformats": "^13.4.1",

     
···

       48
       50
        
           "tailwind-merge": "^3.3.1",

     

       49
       51
        
           "tailwindcss": "4",

     

       50
       52
        
           "tw-animate-css": "^1.4.0",

     

       51
       51
       -
           "zlib": "^1.0.5",

     

       52
       52
       -
           "bun-plugin-tailwind": "^0.1.2"

     

       53
       53
       +
           "zlib": "^1.0.5"

     

       53
       54
        
         },

     

       54
       55
        
         "devDependencies": {

     

       55
       56
        
           "@types/react": "^19.2.2",

+127

apps/main-app/src/lib/ignore-patterns.ts

···

       1
       1
       +
       import ignore, { type Ignore } from 'ignore'

     

       2
       2
       +
       import { readFileSync, existsSync } from 'fs'

     

       3
       3
       +
       import { join } from 'path'

     

       4
       4
       +
       

     

       5
       5
       +
       interface IgnoreConfig {

     

       6
       6
       +
           version: string

     

       7
       7
       +
           description: string

     

       8
       8
       +
           patterns: string[]

     

       9
       9
       +
       }

     

       10
       10
       +
       

     

       11
       11
       +
       /**

     

       12
       12
       +
        * Load default ignore patterns from the .wispignore.json file in the monorepo root

     

       13
       13
       +
        */

     

       14
       14
       +
       function loadDefaultPatterns(): string[] {

     

       15
       15
       +
           try {

     

       16
       16
       +
               // Path to the default ignore patterns JSON file (monorepo root, 3 levels up from this file)

     

       17
       17
       +
               const defaultJsonPath = join(__dirname, '../../../../.wispignore.json')

     

       18
       18
       +
       

     

       19
       19
       +
               if (!existsSync(defaultJsonPath)) {

     

       20
       20
       +
                   console.warn('⚠️  Default .wispignore.json not found, using hardcoded patterns')

     

       21
       21
       +
                   return getHardcodedPatterns()

     

       22
       22
       +
               }

     

       23
       23
       +
       

     

       24
       24
       +
               const contents = readFileSync(defaultJsonPath, 'utf-8')

     

       25
       25
       +
               const config: IgnoreConfig = JSON.parse(contents)

     

       26
       26
       +
               return config.patterns

     

       27
       27
       +
           } catch (error) {

     

       28
       28
       +
               console.error('Failed to load default ignore patterns:', error)

     

       29
       29
       +
               return getHardcodedPatterns()

     

       30
       30
       +
           }

     

       31
       31
       +
       }

     

       32
       32
       +
       

     

       33
       33
       +
       /**

     

       34
       34
       +
        * Hardcoded fallback patterns (same as in .wispignore.json)

     

       35
       35
       +
        */

     

       36
       36
       +
       function getHardcodedPatterns(): string[] {

     

       37
       37
       +
           return [

     

       38
       38
       +
               '.git',

     

       39
       39
       +
               '.git/**',

     

       40
       40
       +
               '.github',

     

       41
       41
       +
               '.github/**',

     

       42
       42
       +
               '.gitlab',

     

       43
       43
       +
               '.gitlab/**',

     

       44
       44
       +
               '.DS_Store',

     

       45
       45
       +
               '.wisp.metadata.json',

     

       46
       46
       +
               '.env',

     

       47
       47
       +
               '.env.*',

     

       48
       48
       +
               'node_modules',

     

       49
       49
       +
               'node_modules/**',

     

       50
       50
       +
               'Thumbs.db',

     

       51
       51
       +
               'desktop.ini',

     

       52
       52
       +
               '._*',

     

       53
       53
       +
               '.Spotlight-V100',

     

       54
       54
       +
               '.Spotlight-V100/**',

     

       55
       55
       +
               '.Trashes',

     

       56
       56
       +
               '.Trashes/**',

     

       57
       57
       +
               '.fseventsd',

     

       58
       58
       +
               '.fseventsd/**',

     

       59
       59
       +
               '.cache',

     

       60
       60
       +
               '.cache/**',

     

       61
       61
       +
               '.temp',

     

       62
       62
       +
               '.temp/**',

     

       63
       63
       +
               '.tmp',

     

       64
       64
       +
               '.tmp/**',

     

       65
       65
       +
               '__pycache__',

     

       66
       66
       +
               '__pycache__/**',

     

       67
       67
       +
               '*.pyc',

     

       68
       68
       +
               '.venv',

     

       69
       69
       +
               '.venv/**',

     

       70
       70
       +
               'venv',

     

       71
       71
       +
               'venv/**',

     

       72
       72
       +
               'env',

     

       73
       73
       +
               'env/**',

     

       74
       74
       +
               '*.swp',

     

       75
       75
       +
               '*.swo',

     

       76
       76
       +
               '*~',

     

       77
       77
       +
               '.tangled',

     

       78
       78
       +
               '.tangled/**',

     

       79
       79
       +
           ]

     

       80
       80
       +
       }

     

       81
       81
       +
       

     

       82
       82
       +
       /**

     

       83
       83
       +
        * Load custom ignore patterns from a .wispignore file

     

       84
       84
       +
        * @param wispignoreContent - Content of the .wispignore file (one pattern per line)

     

       85
       85
       +
        */

     

       86
       86
       +
       function loadWispignorePatterns(wispignoreContent: string): string[] {

     

       87
       87
       +
           return wispignoreContent

     

       88
       88
       +
               .split('\n')

     

       89
       89
       +
               .map(line => line.trim())

     

       90
       90
       +
               .filter(line => line && !line.startsWith('#')) // Skip empty lines and comments

     

       91
       91
       +
       }

     

       92
       92
       +
       

     

       93
       93
       +
       /**

     

       94
       94
       +
        * Create an ignore matcher

     

       95
       95
       +
        * @param customPatterns - Optional custom patterns from a .wispignore file

     

       96
       96
       +
        */

     

       97
       97
       +
       export function createIgnoreMatcher(customPatterns?: string[]): Ignore {

     

       98
       98
       +
           const ig = ignore()

     

       99
       99
       +
       

     

       100
       100
       +
           // Add default patterns

     

       101
       101
       +
           const defaultPatterns = loadDefaultPatterns()

     

       102
       102
       +
           ig.add(defaultPatterns)

     

       103
       103
       +
       

     

       104
       104
       +
           // Add custom patterns if provided

     

       105
       105
       +
           if (customPatterns && customPatterns.length > 0) {

     

       106
       106
       +
               ig.add(customPatterns)

     

       107
       107
       +
               console.log(`Loaded ${customPatterns.length} custom patterns from .wispignore`)

     

       108
       108
       +
           }

     

       109
       109
       +
       

     

       110
       110
       +
           return ig

     

       111
       111
       +
       }

     

       112
       112
       +
       

     

       113
       113
       +
       /**

     

       114
       114
       +
        * Check if a file path should be ignored

     

       115
       115
       +
        * @param matcher - The ignore matcher

     

       116
       116
       +
        * @param filePath - The file path to check (relative to site root)

     

       117
       117
       +
        */

     

       118
       118
       +
       export function shouldIgnore(matcher: Ignore, filePath: string): boolean {

     

       119
       119
       +
           return matcher.ignores(filePath)

     

       120
       120
       +
       }

     

       121
       121
       +
       

     

       122
       122
       +
       /**

     

       123
       123
       +
        * Parse .wispignore content and return patterns

     

       124
       124
       +
        */

     

       125
       125
       +
       export function parseWispignore(content: string): string[] {

     

       126
       126
       +
           return loadWispignorePatterns(content)

     

       127
       127
       +
       }

+22 -97

apps/main-app/src/routes/wisp.ts

···

       34
       34
        
           failUploadJob,

     

       35
       35
        
           addJobListener

     

       36
       36
        
       } from '../lib/upload-jobs'

     

       37
       37
       +
       import { createIgnoreMatcher, shouldIgnore, parseWispignore } from '../lib/ignore-patterns'

     

       38
       38
       +
       import type { Ignore } from 'ignore'

     

       37
       39
        
       

     

       38
       40
        
       const logger = createLogger('main-app')

     

       39
       41
        
       

     
···

       145
       147
        
                   }

     

       146
       148
        
               }

     

       147
       149
        
       

     

       150
       150
       +
               // Check for .wispignore file in uploaded files

     

       151
       151
       +
               let customIgnorePatterns: string[] = [];

     

       152
       152
       +
               const wispignoreFile = fileArray.find(f => f && f.name && f.name.endsWith('.wispignore'));

     

       153
       153
       +
               if (wispignoreFile) {

     

       154
       154
       +
                   try {

     

       155
       155
       +
                       const content = await wispignoreFile.text();

     

       156
       156
       +
                       customIgnorePatterns = parseWispignore(content);

     

       157
       157
       +
                       console.log(`Found .wispignore file with ${customIgnorePatterns.length} custom patterns`);

     

       158
       158
       +
                   } catch (err) {

     

       159
       159
       +
                       console.warn('Failed to parse .wispignore file:', err);

     

       160
       160
       +
                   }

     

       161
       161
       +
               }

     

       162
       162
       +
       

     

       163
       163
       +
               // Create ignore matcher with default and custom patterns

     

       164
       164
       +
               const ignoreMatcher = createIgnoreMatcher(customIgnorePatterns);

     

       165
       165
       +
       

     

       148
       166
        
               // Convert File objects to UploadedFile format

     

       149
       167
        
               const uploadedFiles: UploadedFile[] = [];

     

       150
       168
        
               const skippedFiles: Array<{ name: string; reason: string }> = [];

     
···

       171
       189
        
                       currentFile: file.name

     

       172
       190
        
                   });

     

       173
       191
        
       

     

       174
       174
       -
                   // Skip unwanted files and directories

     

       192
       192
       +
                   // Skip files that match ignore patterns

     

       175
       193
        
                   const normalizedPath = file.name.replace(/^[^\/]*\//, '');

     

       176
       176
       -
                   const fileName = normalizedPath.split('/').pop() || '';

     

       177
       177
       -
                   const pathParts = normalizedPath.split('/');

     

       178
       194
        
       

     

       179
       179
       -
                   // .git directory (version control - thousands of files)

     

       180
       180
       -
                   if (normalizedPath.startsWith('.git/') || normalizedPath === '.git') {

     

       181
       181
       -
                       console.log(`Skipping .git file: ${file.name}`);

     

       182
       182
       -
                       skippedFiles.push({

     

       183
       183
       -
                           name: file.name,

     

       184
       184
       -
                           reason: '.git directory excluded'

     

       185
       185
       -
                       });

     

       186
       186
       -
                       continue;

     

       187
       187
       -
                   }

     

       188
       188
       -
       

     

       189
       189
       -
                   // .DS_Store (macOS metadata - can leak info)

     

       190
       190
       -
                   if (fileName === '.DS_Store') {

     

       191
       191
       -
                       console.log(`Skipping .DS_Store file: ${file.name}`);

     

       195
       195
       +
                   if (shouldIgnore(ignoreMatcher, normalizedPath)) {

     

       196
       196
       +
                       console.log(`Skipping ignored file: ${file.name}`);

     

       192
       197
        
                       skippedFiles.push({

     

       193
       198
        
                           name: file.name,

     

       194
       194
       -
                           reason: '.DS_Store file excluded'

     

       195
       195
       -
                       });

     

       196
       196
       -
                       continue;

     

       197
       197
       -
                   }

     

       198
       198
       -
       

     

       199
       199
       -
                   // .env files (environment variables with secrets)

     

       200
       200
       -
                   if (fileName.startsWith('.env')) {

     

       201
       201
       -
                       console.log(`Skipping .env file: ${file.name}`);

     

       202
       202
       -
                       skippedFiles.push({

     

       203
       203
       -
                           name: file.name,

     

       204
       204
       -
                           reason: 'environment files excluded for security'

     

       205
       205
       -
                       });

     

       206
       206
       -
                       continue;

     

       207
       207
       -
                   }

     

       208
       208
       -
       

     

       209
       209
       -
                   // node_modules (dependency folder - can be 100,000+ files)

     

       210
       210
       -
                   if (pathParts.includes('node_modules')) {

     

       211
       211
       -
                       console.log(`Skipping node_modules file: ${file.name}`);

     

       212
       212
       -
                       skippedFiles.push({

     

       213
       213
       -
                           name: file.name,

     

       214
       214
       -
                           reason: 'node_modules excluded'

     

       215
       215
       -
                       });

     

       216
       216
       -
                       continue;

     

       217
       217
       -
                   }

     

       218
       218
       -
       

     

       219
       219
       -
                   // OS metadata files

     

       220
       220
       -
                   if (fileName === 'Thumbs.db' || fileName === 'desktop.ini' || fileName.startsWith('._')) {

     

       221
       221
       -
                       console.log(`Skipping OS metadata file: ${file.name}`);

     

       222
       222
       -
                       skippedFiles.push({

     

       223
       223
       -
                           name: file.name,

     

       224
       224
       -
                           reason: 'OS metadata file excluded'

     

       225
       225
       -
                       });

     

       226
       226
       -
                       continue;

     

       227
       227
       -
                   }

     

       228
       228
       -
       

     

       229
       229
       -
                   // macOS system directories

     

       230
       230
       -
                   if (pathParts.includes('.Spotlight-V100') || pathParts.includes('.Trashes') || pathParts.includes('.fseventsd')) {

     

       231
       231
       -
                       console.log(`Skipping macOS system file: ${file.name}`);

     

       232
       232
       -
                       skippedFiles.push({

     

       233
       233
       -
                           name: file.name,

     

       234
       234
       -
                           reason: 'macOS system directory excluded'

     

       235
       235
       -
                       });

     

       236
       236
       -
                       continue;

     

       237
       237
       -
                   }

     

       238
       238
       -
       

     

       239
       239
       -
                   // Cache and temp directories

     

       240
       240
       -
                   if (pathParts.some(part => part === '.cache' || part === '.temp' || part === '.tmp')) {

     

       241
       241
       -
                       console.log(`Skipping cache/temp file: ${file.name}`);

     

       242
       242
       -
                       skippedFiles.push({

     

       243
       243
       -
                           name: file.name,

     

       244
       244
       -
                           reason: 'cache/temp directory excluded'

     

       245
       245
       -
                       });

     

       246
       246
       -
                       continue;

     

       247
       247
       -
                   }

     

       248
       248
       -
       

     

       249
       249
       -
                   // Python cache

     

       250
       250
       -
                   if (pathParts.includes('__pycache__') || fileName.endsWith('.pyc')) {

     

       251
       251
       -
                       console.log(`Skipping Python cache file: ${file.name}`);

     

       252
       252
       -
                       skippedFiles.push({

     

       253
       253
       -
                           name: file.name,

     

       254
       254
       -
                           reason: 'Python cache excluded'

     

       255
       255
       -
                       });

     

       256
       256
       -
                       continue;

     

       257
       257
       -
                   }

     

       258
       258
       -
       

     

       259
       259
       -
                   // Python virtual environments

     

       260
       260
       -
                   if (pathParts.some(part => part === '.venv' || part === 'venv' || part === 'env')) {

     

       261
       261
       -
                       console.log(`Skipping Python venv file: ${file.name}`);

     

       262
       262
       -
                       skippedFiles.push({

     

       263
       263
       -
                           name: file.name,

     

       264
       264
       -
                           reason: 'Python virtual environment excluded'

     

       265
       265
       -
                       });

     

       266
       266
       -
                       continue;

     

       267
       267
       -
                   }

     

       268
       268
       -
       

     

       269
       269
       -
                   // Editor swap files

     

       270
       270
       -
                   if (fileName.endsWith('.swp') || fileName.endsWith('.swo') || fileName.endsWith('~')) {

     

       271
       271
       -
                       console.log(`Skipping editor swap file: ${file.name}`);

     

       272
       272
       -
                       skippedFiles.push({

     

       273
       273
       -
                           name: file.name,

     

       274
       274
       -
                           reason: 'editor swap file excluded'

     

       199
       199
       +
                           reason: 'matched ignore pattern'

     

       275
       200
        
                       });

     

       276
       201
        
                       continue;

     

       277
       202
        
                   }

bun.lock

···

       73
       73
        
               "class-variance-authority": "^0.7.1",

     

       74
       74
        
               "clsx": "^2.1.1",

     

       75
       75
        
               "elysia": "latest",

     

       76
       76
       +
               "ignore": "^7.0.5",

     

       76
       77
        
               "iron-session": "^8.0.4",

     

       77
       78
        
               "lucide-react": "^0.546.0",

     

       78
       79
        
               "multiformats": "^13.4.1",

     
···

       768
       769
        
           "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="],

     

       769
       770
        
       

     

       770
       771
        
           "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],

     

       772
       772
       +
       

     

       773
       773
       +
           "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="],

     

       771
       774
        
       

     

       772
       775
        
           "import-in-the-middle": ["import-in-the-middle@1.15.0", "", { "dependencies": { "acorn": "^8.14.0", "acorn-import-attributes": "^1.9.5", "cjs-module-lexer": "^1.2.2", "module-details-from-path": "^1.0.3" } }, "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA=="],

     

       773
       776

+61 -1

cli/Cargo.lock

···

       360
       360
        
       ]

     

       361
       361
        
       

     

       362
       362
        
       [[package]]

     

       363
       363
       +
       name = "bstr"

     

       364
       364
       +
       version = "1.12.1"

     

       365
       365
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       366
       366
       +
       checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"

     

       367
       367
       +
       dependencies = [

     

       368
       368
       +
        "memchr",

     

       369
       369
       +
        "serde",

     

       370
       370
       +
       ]

     

       371
       371
       +
       

     

       372
       372
       +
       [[package]]

     

       363
       373
        
       name = "btree-range-map"

     

       364
       374
        
       version = "0.7.2"

     

       365
       375
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       674
       684
        
       version = "0.5.15"

     

       675
       685
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       676
       686
        
       checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"

     

       687
       687
       +
       dependencies = [

     

       688
       688
       +
        "crossbeam-utils",

     

       689
       689
       +
       ]

     

       690
       690
       +
       

     

       691
       691
       +
       [[package]]

     

       692
       692
       +
       name = "crossbeam-deque"

     

       693
       693
       +
       version = "0.8.6"

     

       694
       694
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       695
       695
       +
       checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"

     

       696
       696
       +
       dependencies = [

     

       697
       697
       +
        "crossbeam-epoch",

     

       698
       698
       +
        "crossbeam-utils",

     

       699
       699
       +
       ]

     

       700
       700
       +
       

     

       701
       701
       +
       [[package]]

     

       702
       702
       +
       name = "crossbeam-epoch"

     

       703
       703
       +
       version = "0.9.18"

     

       704
       704
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       705
       705
       +
       checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"

     

       677
       706
        
       dependencies = [

     

       678
       707
        
        "crossbeam-utils",

     

       679
       708
        
       ]

     
···

       1209
       1238
        
       checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"

     

       1210
       1239
        
       

     

       1211
       1240
        
       [[package]]

     

       1241
       1241
       +
       name = "globset"

     

       1242
       1242
       +
       version = "0.4.18"

     

       1243
       1243
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1244
       1244
       +
       checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"

     

       1245
       1245
       +
       dependencies = [

     

       1246
       1246
       +
        "aho-corasick",

     

       1247
       1247
       +
        "bstr",

     

       1248
       1248
       +
        "log",

     

       1249
       1249
       +
        "regex-automata",

     

       1250
       1250
       +
        "regex-syntax",

     

       1251
       1251
       +
       ]

     

       1252
       1252
       +
       

     

       1253
       1253
       +
       [[package]]

     

       1212
       1254
        
       name = "gloo-storage"

     

       1213
       1255
        
       version = "0.3.0"

     

       1214
       1256
        
       source = "registry+https://github.com/rust-lang/crates.io-index"

     
···

       1649
       1691
        
       dependencies = [

     

       1650
       1692
        
        "icu_normalizer",

     

       1651
       1693
        
        "icu_properties",

     

       1694
       1694
       +
       ]

     

       1695
       1695
       +
       

     

       1696
       1696
       +
       [[package]]

     

       1697
       1697
       +
       name = "ignore"

     

       1698
       1698
       +
       version = "0.4.25"

     

       1699
       1699
       +
       source = "registry+https://github.com/rust-lang/crates.io-index"

     

       1700
       1700
       +
       checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a"

     

       1701
       1701
       +
       dependencies = [

     

       1702
       1702
       +
        "crossbeam-deque",

     

       1703
       1703
       +
        "globset",

     

       1704
       1704
       +
        "log",

     

       1705
       1705
       +
        "memchr",

     

       1706
       1706
       +
        "regex-automata",

     

       1707
       1707
       +
        "same-file",

     

       1708
       1708
       +
        "walkdir",

     

       1709
       1709
       +
        "winapi-util",

     

       1652
       1710
        
       ]

     

       1653
       1711
        
       

     

       1654
       1712
        
       [[package]]

     
···

       4939
       4997
        
       

     

       4940
       4998
        
       [[package]]

     

       4941
       4999
        
       name = "wisp-cli"

     

       4942
       4942
       -
       version = "0.4.1"

     

       5000
       5000
       +
       version = "0.4.2"

     

       4943
       5001
        
       dependencies = [

     

       4944
       5002
        
        "axum",

     

       4945
       5003
        
        "base64 0.22.1",

     
···

       4948
       5006
        
        "clap",

     

       4949
       5007
        
        "flate2",

     

       4950
       5008
        
        "futures",

     

       5009
       5009
       +
        "globset",

     

       5010
       5010
       +
        "ignore",

     

       4951
       5011
        
        "jacquard",

     

       4952
       5012
        
        "jacquard-api",

     

       4953
       5013
        
        "jacquard-common",

+3 -1

cli/Cargo.toml

···

       1
       1
        
       [package]

     

       2
       2
        
       name = "wisp-cli"

     

       3
       3
       -
       version = "0.4.1"

     

       3
       3
       +
       version = "0.4.2"

     

       4
       4
        
       edition = "2024"

     

       5
       5
        
       

     

       6
       6
        
       [features]

     
···

       40
       40
        
       chrono = "0.4"

     

       41
       41
        
       url = "2.5"

     

       42
       42
        
       regex = "1.11"

     

       43
       43
       +
       ignore = "0.4"

     

       44
       44
       +
       globset = "0.4"

+149

cli/src/ignore_patterns.rs

···

       1
       1
       +
       use globset::{Glob, GlobSet, GlobSetBuilder};

     

       2
       2
       +
       use serde::{Deserialize, Serialize};

     

       3
       3
       +
       use std::path::Path;

     

       4
       4
       +
       use miette::IntoDiagnostic;

     

       5
       5
       +
       

     

       6
       6
       +
       #[derive(Debug, Deserialize, Serialize)]

     

       7
       7
       +
       struct IgnoreConfig {

     

       8
       8
       +
           patterns: Vec<String>,

     

       9
       9
       +
       }

     

       10
       10
       +
       

     

       11
       11
       +
       /// Load ignore patterns from the default .wispignore.json file

     

       12
       12
       +
       fn load_default_patterns() -> miette::Result<Vec<String>> {

     

       13
       13
       +
           // Path to the default ignore patterns JSON file (in the monorepo root)

     

       14
       14
       +
           let default_json_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../.wispignore.json");

     

       15
       15
       +
       

     

       16
       16
       +
           match std::fs::read_to_string(default_json_path) {

     

       17
       17
       +
               Ok(contents) => {

     

       18
       18
       +
                   let config: IgnoreConfig = serde_json::from_str(&contents).into_diagnostic()?;

     

       19
       19
       +
                   Ok(config.patterns)

     

       20
       20
       +
               }

     

       21
       21
       +
               Err(_) => {

     

       22
       22
       +
                   // If the default file doesn't exist, return hardcoded patterns as fallback

     

       23
       23
       +
                   eprintln!("⚠️  Default .wispignore.json not found, using hardcoded patterns");

     

       24
       24
       +
                   Ok(get_hardcoded_patterns())

     

       25
       25
       +
               }

     

       26
       26
       +
           }

     

       27
       27
       +
       }

     

       28
       28
       +
       

     

       29
       29
       +
       /// Hardcoded fallback patterns (same as in .wispignore.json)

     

       30
       30
       +
       fn get_hardcoded_patterns() -> Vec<String> {

     

       31
       31
       +
           vec![

     

       32
       32
       +
               ".git".to_string(),

     

       33
       33
       +
               ".git/**".to_string(),

     

       34
       34
       +
               ".github".to_string(),

     

       35
       35
       +
               ".github/**".to_string(),

     

       36
       36
       +
               ".gitlab".to_string(),

     

       37
       37
       +
               ".gitlab/**".to_string(),

     

       38
       38
       +
               ".DS_Store".to_string(),

     

       39
       39
       +
               ".wisp.metadata.json".to_string(),

     

       40
       40
       +
               ".env".to_string(),

     

       41
       41
       +
               ".env.*".to_string(),

     

       42
       42
       +
               "node_modules".to_string(),

     

       43
       43
       +
               "node_modules/**".to_string(),

     

       44
       44
       +
               "Thumbs.db".to_string(),

     

       45
       45
       +
               "desktop.ini".to_string(),

     

       46
       46
       +
               "._*".to_string(),

     

       47
       47
       +
               ".Spotlight-V100".to_string(),

     

       48
       48
       +
               ".Spotlight-V100/**".to_string(),

     

       49
       49
       +
               ".Trashes".to_string(),

     

       50
       50
       +
               ".Trashes/**".to_string(),

     

       51
       51
       +
               ".fseventsd".to_string(),

     

       52
       52
       +
               ".fseventsd/**".to_string(),

     

       53
       53
       +
               ".cache".to_string(),

     

       54
       54
       +
               ".cache/**".to_string(),

     

       55
       55
       +
               ".temp".to_string(),

     

       56
       56
       +
               ".temp/**".to_string(),

     

       57
       57
       +
               ".tmp".to_string(),

     

       58
       58
       +
               ".tmp/**".to_string(),

     

       59
       59
       +
               "__pycache__".to_string(),

     

       60
       60
       +
               "__pycache__/**".to_string(),

     

       61
       61
       +
               "*.pyc".to_string(),

     

       62
       62
       +
               ".venv".to_string(),

     

       63
       63
       +
               ".venv/**".to_string(),

     

       64
       64
       +
               "venv".to_string(),

     

       65
       65
       +
               "venv/**".to_string(),

     

       66
       66
       +
               "env".to_string(),

     

       67
       67
       +
               "env/**".to_string(),

     

       68
       68
       +
               "*.swp".to_string(),

     

       69
       69
       +
               "*.swo".to_string(),

     

       70
       70
       +
               "*~".to_string(),

     

       71
       71
       +
               ".tangled".to_string(),

     

       72
       72
       +
               ".tangled/**".to_string(),

     

       73
       73
       +
           ]

     

       74
       74
       +
       }

     

       75
       75
       +
       

     

       76
       76
       +
       /// Load custom ignore patterns from a .wispignore file in the given directory

     

       77
       77
       +
       fn load_wispignore_file(dir_path: &Path) -> miette::Result<Vec<String>> {

     

       78
       78
       +
           let wispignore_path = dir_path.join(".wispignore");

     

       79
       79
       +
       

     

       80
       80
       +
           if !wispignore_path.exists() {

     

       81
       81
       +
               return Ok(Vec::new());

     

       82
       82
       +
           }

     

       83
       83
       +
       

     

       84
       84
       +
           let contents = std::fs::read_to_string(&wispignore_path).into_diagnostic()?;

     

       85
       85
       +
       

     

       86
       86
       +
           // Parse gitignore-style file (one pattern per line, # for comments)

     

       87
       87
       +
           let patterns: Vec<String> = contents

     

       88
       88
       +
               .lines()

     

       89
       89
       +
               .filter_map(|line| {

     

       90
       90
       +
                   let line = line.trim();

     

       91
       91
       +
                   // Skip empty lines and comments

     

       92
       92
       +
                   if line.is_empty() || line.starts_with('#') {

     

       93
       93
       +
                       None

     

       94
       94
       +
                   } else {

     

       95
       95
       +
                       Some(line.to_string())

     

       96
       96
       +
                   }

     

       97
       97
       +
               })

     

       98
       98
       +
               .collect();

     

       99
       99
       +
       

     

       100
       100
       +
           if !patterns.is_empty() {

     

       101
       101
       +
               println!("Loaded {} custom patterns from .wispignore", patterns.len());

     

       102
       102
       +
           }

     

       103
       103
       +
       

     

       104
       104
       +
           Ok(patterns)

     

       105
       105
       +
       }

     

       106
       106
       +
       

     

       107
       107
       +
       /// Build a GlobSet from a list of patterns

     

       108
       108
       +
       fn build_globset(patterns: Vec<String>) -> miette::Result<GlobSet> {

     

       109
       109
       +
           let mut builder = GlobSetBuilder::new();

     

       110
       110
       +
       

     

       111
       111
       +
           for pattern in patterns {

     

       112
       112
       +
               let glob = Glob::new(&pattern).into_diagnostic()?;

     

       113
       113
       +
               builder.add(glob);

     

       114
       114
       +
           }

     

       115
       115
       +
       

     

       116
       116
       +
           let globset = builder.build().into_diagnostic()?;

     

       117
       117
       +
           Ok(globset)

     

       118
       118
       +
       }

     

       119
       119
       +
       

     

       120
       120
       +
       /// IgnoreMatcher handles checking if paths should be ignored

     

       121
       121
       +
       pub struct IgnoreMatcher {

     

       122
       122
       +
           globset: GlobSet,

     

       123
       123
       +
       }

     

       124
       124
       +
       

     

       125
       125
       +
       impl IgnoreMatcher {

     

       126
       126
       +
           /// Create a new IgnoreMatcher for the given directory

     

       127
       127
       +
           /// Loads default patterns and any custom .wispignore file

     

       128
       128
       +
           pub fn new(dir_path: &Path) -> miette::Result<Self> {

     

       129
       129
       +
               let mut all_patterns = load_default_patterns()?;

     

       130
       130
       +
       

     

       131
       131
       +
               // Load custom patterns from .wispignore

     

       132
       132
       +
               let custom_patterns = load_wispignore_file(dir_path)?;

     

       133
       133
       +
               all_patterns.extend(custom_patterns);

     

       134
       134
       +
       

     

       135
       135
       +
               let globset = build_globset(all_patterns)?;

     

       136
       136
       +
       

     

       137
       137
       +
               Ok(Self { globset })

     

       138
       138
       +
           }

     

       139
       139
       +
       

     

       140
       140
       +
           /// Check if the given path (relative to site root) should be ignored

     

       141
       141
       +
           pub fn is_ignored(&self, path: &str) -> bool {

     

       142
       142
       +
               self.globset.is_match(path)

     

       143
       143
       +
           }

     

       144
       144
       +
       

     

       145
       145
       +
           /// Check if a filename should be ignored (checks just the filename, not full path)

     

       146
       146
       +
           pub fn is_filename_ignored(&self, filename: &str) -> bool {

     

       147
       147
       +
               self.globset.is_match(filename)

     

       148
       148
       +
           }

     

       149
       149
       +
       }

+14 -56

cli/src/main.rs

···

       8
       8
        
       mod serve;

     

       9
       9
        
       mod subfs_utils;

     

       10
       10
        
       mod redirects;

     

       11
       11
       +
       mod ignore_patterns;

     

       11
       12
        
       

     

       12
       13
        
       use clap::{Parser, Subcommand};

     

       13
       14
        
       use jacquard::CowStr;

     
···

       323
       324
        
               }

     

       324
       325
        
           };

     

       325
       326
        
       

     

       326
       326
       -
           // Build directory tree

     

       327
       327
       -
           let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new()).await?;

     

       327
       327
       +
           // Build directory tree with ignore patterns

     

       328
       328
       +
           let ignore_matcher = ignore_patterns::IgnoreMatcher::new(&path)?;

     

       329
       329
       +
           let (root_dir, total_files, reused_count) = build_directory(agent, &path, &existing_blob_map, String::new(), &ignore_matcher).await?;

     

       328
       330
        
           let uploaded_count = total_files - reused_count;

     

       329
       331
        
       

     

       330
       332
        
           // Check if we need to split into subfs records

     
···

       603
       605
        
           dir_path: &'a Path,

     

       604
       606
        
           existing_blobs: &'a HashMap<String, (jacquard_common::types::blob::BlobRef<'static>, String)>,

     

       605
       607
        
           current_path: String,

     

       608
       608
       +
           ignore_matcher: &'a ignore_patterns::IgnoreMatcher,

     

       606
       609
        
       ) -> std::pin::Pin<Box<dyn std::future::Future<Output = miette::Result<(Directory<'static>, usize, usize)>> + 'a>>

     

       607
       610
        
       {

     

       608
       611
        
           Box::pin(async move {

     
···

       623
       626
        
                   .ok_or_else(|| miette::miette!("Invalid filename: {:?}", name))?

     

       624
       627
        
                   .to_string();

     

       625
       628
        
       

     

       626
       626
       -
               // Skip unwanted files and directories

     

       627
       627
       -
       

     

       628
       628
       -
               // .git directory (version control - thousands of files)

     

       629
       629
       -
               if name_str == ".git" {

     

       630
       630
       -
                   continue;

     

       631
       631
       -
               }

     

       632
       632
       -
       

     

       633
       633
       -
               // .DS_Store (macOS metadata - can leak info)

     

       634
       634
       -
               if name_str == ".DS_Store" {

     

       635
       635
       -
                   continue;

     

       636
       636
       -
               }

     

       637
       637
       -
       

     

       638
       638
       -
               // .wisp.metadata.json (wisp internal metadata - should not be uploaded)

     

       639
       639
       -
               if name_str == ".wisp.metadata.json" {

     

       640
       640
       -
                   continue;

     

       641
       641
       -
               }

     

       642
       642
       -
       

     

       643
       643
       -
               // .env files (environment variables with secrets)

     

       644
       644
       -
               if name_str.starts_with(".env") {

     

       645
       645
       -
                   continue;

     

       646
       646
       -
               }

     

       647
       647
       -
       

     

       648
       648
       -
               // node_modules (dependency folder - can be 100,000+ files)

     

       649
       649
       -
               if name_str == "node_modules" {

     

       650
       650
       -
                   continue;

     

       651
       651
       -
               }

     

       652
       652
       -
       

     

       653
       653
       -
               // OS metadata files

     

       654
       654
       -
               if name_str == "Thumbs.db" || name_str == "desktop.ini" || name_str.starts_with("._") {

     

       655
       655
       -
                   continue;

     

       656
       656
       -
               }

     

       629
       629
       +
               // Construct full path for ignore checking

     

       630
       630
       +
               let full_path = if current_path.is_empty() {

     

       631
       631
       +
                   name_str.clone()

     

       632
       632
       +
               } else {

     

       633
       633
       +
                   format!("{}/{}", current_path, name_str)

     

       634
       634
       +
               };

     

       657
       635
        
       

     

       658
       658
       -
               // macOS system directories

     

       659
       659
       -
               if name_str == ".Spotlight-V100" || name_str == ".Trashes" || name_str == ".fseventsd" {

     

       660
       660
       -
                   continue;

     

       661
       661
       -
               }

     

       662
       662
       -
       

     

       663
       663
       -
               // Cache and temp directories

     

       664
       664
       -
               if name_str == ".cache" || name_str == ".temp" || name_str == ".tmp" {

     

       665
       665
       -
                   continue;

     

       666
       666
       -
               }

     

       667
       667
       -
       

     

       668
       668
       -
               // Python cache

     

       669
       669
       -
               if name_str == "__pycache__" || name_str.ends_with(".pyc") {

     

       670
       670
       -
                   continue;

     

       671
       671
       -
               }

     

       672
       672
       -
       

     

       673
       673
       -
               // Python virtual environments

     

       674
       674
       -
               if name_str == ".venv" || name_str == "venv" || name_str == "env" {

     

       675
       675
       -
                   continue;

     

       676
       676
       -
               }

     

       677
       677
       -
       

     

       678
       678
       -
               // Editor swap files

     

       679
       679
       -
               if name_str.ends_with(".swp") || name_str.ends_with(".swo") || name_str.ends_with("~") {

     

       636
       636
       +
               // Skip files/directories that match ignore patterns

     

       637
       637
       +
               if ignore_matcher.is_ignored(&full_path) || ignore_matcher.is_filename_ignored(&name_str) {

     

       680
       638
        
                   continue;

     

       681
       639
        
               }

     

       682
       640
        
       

     
···

       732
       690
        
               } else {

     

       733
       691
        
                   format!("{}/{}", current_path, name)

     

       734
       692
        
               };

     

       735
       735
       -
               let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path).await?;

     

       693
       693
       +
               let (subdir, sub_total, sub_reused) = build_directory(agent, &path, existing_blobs, subdir_path, ignore_matcher).await?;

     

       736
       694
        
               dir_entries.push(Entry::new()

     

       737
       695
        
                   .name(CowStr::from(name))

     

       738
       696
        
                   .node(EntryNode::Directory(Box::new(subdir)))