wip library to store cold objects in s3, warm objects on disk, and hot objects in memory
nekomimi.pet
nodejs
typescript
1# Tiered Storage Configuration
2# Copy this file to .env and configure for your environment
3
4# ============================================================================
5# S3 Configuration (Cold Tier - Required)
6# ============================================================================
7
8# AWS S3 bucket name (or S3-compatible bucket)
9S3_BUCKET=tiered-storage-cache
10
11# Optional: Separate bucket for metadata (RECOMMENDED for production!)
12# When set, metadata is stored as separate JSON objects instead of S3 object metadata.
13# This allows fast, cheap metadata updates without copying entire objects.
14# Leave blank to store metadata in S3 object metadata fields (slower, more expensive).
15S3_METADATA_BUCKET=tiered-storage-metadata
16
17# AWS region
18S3_REGION=us-east-1
19
20# S3 endpoint (optional - for S3-compatible services like R2, Minio)
21# Leave blank for AWS S3
22# For Cloudflare R2: https://YOUR-ACCOUNT-ID.r2.cloudflarestorage.com
23# For MinIO: http://localhost:9000
24# For other S3-compatible: https://s3.your-service.com
25# S3_ENDPOINT=
26
27# Force path-style URLs (usually needed for S3-compatible services)
28# Default: true (recommended for most S3-compatible services)
29# Set to false only if your service requires virtual-host-style URLs
30# S3_FORCE_PATH_STYLE=true
31
32# AWS credentials
33# If not provided, uses default AWS credential chain
34# (environment variables, ~/.aws/credentials, IAM roles, etc.)
35AWS_ACCESS_KEY_ID=your_access_key_id
36AWS_SECRET_ACCESS_KEY=your_secret_access_key
37
38# ============================================================================
39# Cloudflare R2 Example Configuration
40# ============================================================================
41# Uncomment these to use Cloudflare R2 instead of AWS S3:
42#
43# S3_BUCKET=my-r2-bucket
44# S3_METADATA_BUCKET=my-r2-metadata-bucket
45# S3_REGION=auto
46# S3_ENDPOINT=https://YOUR-ACCOUNT-ID.r2.cloudflarestorage.com
47# AWS_ACCESS_KEY_ID=your_r2_access_key_id
48# AWS_SECRET_ACCESS_KEY=your_r2_secret_access_key
49
50# ============================================================================
51# Memory Tier Configuration (Hot)
52# ============================================================================
53
54# Maximum size in bytes for hot (memory) tier
55# Default: 100MB
56MEMORY_MAX_SIZE_BYTES=104857600
57
58# Maximum number of items in hot tier
59# Optional - if not set, only size limit applies
60MEMORY_MAX_ITEMS=1000
61
62# ============================================================================
63# Disk Tier Configuration (Warm)
64# ============================================================================
65
66# Directory for warm tier cache
67# Default: ./cache/warm
68DISK_WARM_DIRECTORY=./cache/warm
69
70# Maximum size in bytes for warm tier
71# Optional - if not set, no size limit
72DISK_WARM_MAX_SIZE_BYTES=10737418240
73
74# Eviction policy when size limit reached
75# Options: lru, fifo, size
76# Default: lru
77DISK_WARM_EVICTION_POLICY=lru
78
79# ============================================================================
80# Storage Options
81# ============================================================================
82
83# Enable compression (gzip)
84# Default: false
85COMPRESSION_ENABLED=true
86
87# Default TTL in milliseconds
88# Optional - if not set, data never expires
89# Example: 1209600000 = 14 days
90DEFAULT_TTL_MS=1209600000
91
92# Promotion strategy: 'eager' or 'lazy'
93# eager: Automatically promote data to upper tiers on read
94# lazy: Only promote on explicit bootstrap or write
95# Default: lazy
96PROMOTION_STRATEGY=lazy
97
98# ============================================================================
99# Bootstrap Configuration
100# ============================================================================
101
102# Number of items to load into hot tier on bootstrap
103# Optional - if not set, loads all items
104BOOTSTRAP_HOT_LIMIT=1000
105
106# Number of days to look back when bootstrapping warm tier
107# Example: 7 = only load items accessed in last 7 days
108BOOTSTRAP_WARM_DAYS=7
109
110# Maximum items to load into warm tier on bootstrap
111# Optional - if not set, loads all matching items
112BOOTSTRAP_WARM_LIMIT=10000
113
114# ============================================================================
115# Performance Tuning
116# ============================================================================
117
118# Maximum concurrent operations for bootstrap
119# Default: 10
120BOOTSTRAP_CONCURRENCY=10
121
122# Timeout for tier operations in milliseconds
123# Default: 30000 (30 seconds)
124TIER_OPERATION_TIMEOUT_MS=30000
125
126# ============================================================================
127# Monitoring & Observability
128# ============================================================================
129
130# Enable statistics tracking
131# Default: true
132STATS_ENABLED=true
133
134# Log level: debug, info, warn, error
135# Default: info
136LOG_LEVEL=info