diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edab244..2f7a003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,26 +47,13 @@ jobs: restore-keys: | ${{ runner.os }}-emscripten- - - name: Cache OpenSSL - uses: actions/cache@v3 - with: - path: openssl-wasm - key: ${{ runner.os }}-openssl-3.3.2-wasm - restore-keys: | - ${{ runner.os }}-openssl- - - - name: Build OpenSSL + - name: Build WASM: OpenSSL + SQLCipher run: | - if [ ! -d "openssl-wasm/lib" ]; then - echo "Building OpenSSL for WASM..." - nix develop --command bash -c "./build-openssl.sh" - else - echo "Using cached OpenSSL" - fi - - - name: Build WASM + nix develop --command bash -c "./build-openssl.sh && ./build-wasm.sh" + + - name: Build WebGL: OpenSSL + SQLCipher run: | - nix develop --command bash -c "./build.sh" + nix develop .#webgl --command bash -c "./build-openssl.sh && ./build-webgl.sh" - name: Prepare cross-platform test run: | @@ -86,14 +73,21 @@ jobs: test -f dist/sqlcipher.cjs test -f dist/sqlcipher.mjs test -f dist/sqlcipher.wasm + test -f target/results/sqlcipher-wasm.zip + test -f target/results/sqlcipher-webgl.zip - - name: Upload build artifacts + - name: Upload WASM archive uses: actions/upload-artifact@v4 with: - name: wasm-build - path: | - dist/ - lib/ + name: sqlcipher-wasm + path: target/results/sqlcipher-wasm.zip + retention-days: 30 + + - name: Upload WebGL archive + uses: actions/upload-artifact@v4 + with: + name: sqlcipher-webgl + path: target/results/sqlcipher-webgl.zip retention-days: 30 - name: Upload test results @@ -121,7 +115,6 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - dist/sqlcipher.cjs - dist/sqlcipher.mjs - dist/sqlcipher.wasm + target/results/sqlcipher-wasm.zip + target/results/sqlcipher-webgl.zip generate_release_notes: true diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 099a05d..a3c7801 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -48,26 +48,9 @@ jobs: restore-keys: | ${{ runner.os }}-emscripten- - - name: Cache OpenSSL - uses: actions/cache@v3 - with: - path: openssl-wasm - key: ${{ runner.os }}-openssl-3.3.2-wasm - restore-keys: | - ${{ runner.os }}-openssl- - - - name: Build OpenSSL - run: | - if [ ! -d "openssl-wasm/lib" ]; then - echo "Building OpenSSL for WASM..." - nix develop --command bash -c "./build-openssl.sh" - else - echo "Using cached OpenSSL" - fi - - name: Quick build test run: | - nix develop --command bash -c "./build.sh" + nix develop --command bash -c "./build-openssl.sh && ./build-wasm.sh" ls -lh dist/ quick-test: @@ -97,14 +80,14 @@ jobs: - name: Cache OpenSSL uses: actions/cache@v3 with: - path: openssl-wasm + path: target/results/openssl key: ${{ runner.os }}-openssl-3.3.2-wasm restore-keys: | ${{ runner.os }}-openssl- - name: Build OpenSSL run: | - if [ ! -d "openssl-wasm/lib" ]; then + if [ ! -d "target/results/openssl/lib" ]; then echo "Building OpenSSL for WASM..." nix develop --command bash -c "./build-openssl.sh" else @@ -113,7 +96,7 @@ jobs: - name: Build run: | - nix develop --command bash -c "./build.sh" + nix develop --command bash -c "./build-openssl.sh && ./build-wasm.sh" - name: Prepare cross-platform test run: | @@ -149,14 +132,14 @@ jobs: - name: Cache OpenSSL uses: actions/cache@v3 with: - path: openssl-wasm + path: target/results/openssl key: ${{ runner.os }}-openssl-3.3.2-wasm restore-keys: | ${{ runner.os }}-openssl- - name: Build OpenSSL run: | - if [ ! -d "openssl-wasm/lib" ]; then + if [ ! -d "target/results/openssl/lib" ]; then echo "Building OpenSSL for WASM..." nix develop --command bash -c "./build-openssl.sh" else @@ -165,7 +148,7 @@ jobs: - name: Build run: | - nix develop --command bash -c "./build.sh" + nix develop --command bash -c "./build-openssl.sh && ./build-wasm.sh" - name: Check bundle sizes run: | diff --git a/.gitignore b/.gitignore index 5abf806..06a2315 100644 --- a/.gitignore +++ b/.gitignore @@ -2,16 +2,11 @@ test-db.html # Build artifacts -build/ +target/ dist/ *.wasm *.js.mem -# OpenSSL build artifacts -openssl-*/ -openssl-wasm/ -*.tar.gz - # Emscripten cache .emscripten-cache/ diff --git a/README.md b/README.md index 7aaefcd..fe6564d 100644 --- a/README.md +++ b/README.md @@ -40,56 +40,82 @@ Add to your `~/.config/nix/nix.conf` (or `/etc/nix/nix.conf`): experimental-features = nix-command flakes ``` -## Quick Start +## Build Variants -1. **Clone the repository** +Two build targets are available, both using SQLCipher v4.9.0 with OpenSSL 3.3.2 encryption: -2. **Enter the Nix environment**: +| Target | Script | Shell | Emscripten | Output | +|--------|--------|-------|------------|--------| +| **WASM** (Node.js/Browser) | `build-wasm.sh` | `nix develop` | Latest | `dist/` + `target/results/sqlcipher-wasm.zip` | +| **WebGL** (Unity) | `build-webgl.sh` | `nix develop .#webgl` | 3.1.10 (pinned for Unity 2022.3.22f) | `target/results/sqlcipher-webgl.zip` | - ```bash - nix develop - ``` +The WebGL build additionally enables `SQLITE_ENABLE_SNAPSHOT` and `SQLITE_ENABLE_COLUMN_METADATA`. - Or with direnv: - ```bash - direnv allow - ``` +## Quick Start -3. **Build OpenSSL for WebAssembly** (first time only): +### WASM (Node.js / Browser) - ```bash - ./build-openssl.sh - ``` +Interactive: -4. **Build SQLCipher WASM**: +```bash +nix develop +./build-openssl.sh +./build-wasm.sh +npm test +``` - ```bash - ./build.sh - ``` +One-liner: -5. **Run all tests**: +```bash +nix develop --command bash -c './build-openssl.sh && ./build-wasm.sh' +``` - ```bash - npm test - ``` +### WebGL (Unity) -6. **Run benchmarks**: +Interactive: - ```bash - npm run bench - ``` +```bash +nix develop .#webgl +./build-openssl.sh +./build-webgl.sh +``` + +One-liner: + +```bash +nix develop .#webgl --command bash -c './build-openssl.sh && ./build-webgl.sh' +``` ## Project Structure ``` . -├── flake.nix # Nix flake configuration +├── flake.nix # Nix flake (default + webgl shells) ├── .envrc # direnv configuration -├── build-openssl.sh # OpenSSL build script -├── build.sh # SQLCipher WASM build script +├── lib.sh # Shared build utilities (includes patch_amalgamation) +├── build-openssl.sh # OpenSSL WASM build script +├── build-wasm.sh # SQLCipher WASM build (CJS/ESM) +├── build-webgl.sh # SQLCipher static lib build (Unity WebGL) ├── package.json # NPM package configuration -├── dist/ # Output directory -│ ├── sqlcipher.js # JavaScript loader +├── target/ +│ ├── build/ # Intermediate build artifacts +│ │ ├── openssl-3.3.2/ # OpenSSL source (shared) +│ │ └── emcc-/ # Per-emscripten-version builds +│ │ ├── openssl-wasm/ # Compiled OpenSSL +│ │ ├── sqlcipher-wasm/ # WASM build working dir +│ │ └── sqlcipher-webgl/ # WebGL build working dir +│ └── results/ # Final build outputs +│ ├── sqlcipher-wasm/ # WASM artifacts (unpacked) +│ ├── sqlcipher-wasm.zip # WASM archive +│ ├── sqlcipher-webgl/ # WebGL artifacts (unpacked) +│ │ ├── libsqlcipher.a +│ │ └── openssl/ +│ │ ├── libcrypto.a +│ │ └── libssl.a +│ └── sqlcipher-webgl.zip # WebGL archive +├── dist/ # NPM package output +│ ├── sqlcipher.cjs # CommonJS module (Node.js) +│ ├── sqlcipher.mjs # ES module (browser) │ └── sqlcipher.wasm # WebAssembly binary ├── lib/ │ └── sqlite-api.cjs # High-level JavaScript API @@ -99,54 +125,64 @@ experimental-features = nix-command flakes │ ├── e2e-test.cjs # End-to-end tests │ ├── file-db-test.cjs # File persistence tests │ ├── encryption-test.cjs # Encryption tests -│ └── cross-platform-db-test.cjs # C++ ↔ WASM compatibility (generated) +│ └── cross-platform-db-test.cjs # C++ <-> WASM compatibility (generated) ├── bench/ │ └── benchmark.cjs # Performance benchmarks ├── examples/ │ └── example.cjs # Usage examples ├── tools/ -│ └── prepare-cross-platform-test.sh # Generate cross-platform test +│ └── prepare-cross-platform-test.sh └── docs/ - └── archive/ # Historical documentation + └── archive/ ``` ## Build Process +All build scripts source `lib.sh` which auto-detects the emscripten version and sets up versioned build/cache directories. This allows the default and webgl shells to coexist without conflicts. + ### 1. OpenSSL Build (`build-openssl.sh`) Downloads and compiles OpenSSL 3.3.2 to WebAssembly: - Configured for WASM target (`linux-generic32`) -- Optimized build (`-O3`) +- Optimized build (`-O3 -flto`) - Disabled features: ASM, threads, engines, hardware acceleration -- Static library output +- Static library output to `target/build/emcc-/openssl-wasm/` +- OpenSSL source tarball is shared across emscripten versions -### 2. SQLCipher Build (`build.sh`) +### 2. SQLCipher WASM Build (`build-wasm.sh`) -Compiles SQLCipher with OpenSSL: +Compiles SQLCipher with OpenSSL into CJS/ESM modules: -1. Copies SQLCipher source from Nix store +1. Copies SQLCipher v4.9.0 source from Nix store 2. Configures and creates amalgamation (`sqlite3.c`) -3. Compiles with OpenSSL crypto provider -4. Links with OpenSSL static libraries -5. Outputs `sqlcipher.js` and `sqlcipher.wasm` +3. Patches amalgamation to use OpenSSL crypto provider +4. Compiles and links with OpenSSL static libraries +5. Outputs to `dist/` (for npm) and `target/results/sqlcipher-wasm/` +6. Creates `target/results/sqlcipher-wasm.zip` -**Key Compilation Flags**: +### 3. SQLCipher WebGL Build (`build-webgl.sh`) + +Compiles SQLCipher into a static library for Unity 2022.3.22f WebGL: + +1. Same amalgamation process as WASM build +2. Compiles with emscripten 3.1.10 (ABI-compatible with Unity's bundled emscripten) +3. Outputs `libsqlcipher.a` + OpenSSL libs to `target/results/sqlcipher-webgl/` +4. Creates `target/results/sqlcipher-webgl.zip` + +**Compilation Flags** (shared by both builds): -SQLCipher flags: - `SQLITE_HAS_CODEC` - Enable encryption - `SQLCIPHER_CRYPTO_OPENSSL` - Use OpenSSL crypto provider - `SQLITE_TEMP_STORE=2` - Use memory for temporary storage -- `SQLITE_THREADSAFE=0` - Disable threading (not needed in WASM) +- `SQLITE_THREADSAFE=1` - Thread-safe (required by SQLCipher v4.9.0) - `SQLITE_ENABLE_FTS5` - Full-text search - `SQLITE_ENABLE_RTREE` - Spatial indexing - `SQLITE_ENABLE_JSON1` - JSON support -Emscripten flags: -- `INITIAL_MEMORY=16MB` - Starting memory -- `MAXIMUM_MEMORY=2GB` - Maximum allowed memory -- `ALLOW_MEMORY_GROWTH=1` - Dynamic memory growth -- `ENVIRONMENT=node,web` - Node.js and browser support +WebGL build adds: +- `SQLITE_ENABLE_SNAPSHOT` - Database snapshot support +- `SQLITE_ENABLE_COLUMN_METADATA` - Column metadata API ## Usage @@ -291,7 +327,7 @@ The test suite includes 5 comprehensive test suites: - API key vs PRAGMA key equivalence ### 5. Cross-Platform Tests (`test/cross-platform-db-test.cjs`) -- C++ (native SQLCipher) → WASM compatibility +- C++ (native SQLCipher) -> WASM compatibility - Database created with native SQLCipher, read by WASM - Binary compatibility verification - Real-world migration scenarios @@ -306,43 +342,6 @@ Run tests in watch mode: npm run test:watch ``` -### Test Output - -``` -╔════════════════════════════════════════════════════════════╗ -║ SQLCipher WASM Test Suite ║ -╚════════════════════════════════════════════════════════════╝ - -Running 5 test suites... - -▶ Running Unit Tests... - Core SQLCipher functionality tests - -✓ Unit Tests completed in 542ms - -▶ Running End-to-End Tests... - Complete workflow tests - -✓ End-to-End Tests completed in 498ms - -... - -╔════════════════════════════════════════════════════════════╗ -║ Test Summary ║ -╚════════════════════════════════════════════════════════════╝ - - ✓ PASS Unit Tests 542ms - ✓ PASS End-to-End Tests 498ms - ✓ PASS File Database Tests 523ms - ✓ PASS Encryption Tests 445ms - ✓ PASS Cross-Platform Tests 389ms - -───────────────────────────────────────────────────────────── - ALL TESTS PASSED - Total: 5 Passed: 5 Failed: 0 - Time: 2.40s -``` - ## Benchmarks Run performance benchmarks: @@ -391,24 +390,24 @@ The reverse also works - databases created in WASM can be used in native applica ### Rebuilding -After changing build scripts: +Clean rebuild (WASM): ```bash -./build.sh +rm -rf target/ dist/ +./build-openssl.sh && ./build-wasm.sh ``` -Clean rebuild: +Clean rebuild (WebGL): ```bash -rm -rf build/ dist/ -./build.sh +rm -rf target/ +./build-openssl.sh && ./build-webgl.sh ``` -Rebuild OpenSSL (rarely needed): +Full clean: ```bash -rm -rf openssl-wasm/ openssl-3.3.2/ -./build-openssl.sh +rm -rf target/ dist/ ``` ### Adding Tests @@ -434,13 +433,12 @@ GitHub Actions workflows: - **CI/CD** (`.github/workflows/ci.yml`) - Build, test, and publish on releases - **PR Checks** (`.github/workflows/pr-check.yml`) - Quick validation on pull requests -Both workflows: -1. Cache OpenSSL and Emscripten for faster builds -2. Build OpenSSL if not cached -3. Build SQLCipher WASM -4. Generate cross-platform test -5. Run all test suites -6. Run benchmarks (CI only) +CI pipeline: +1. Build OpenSSL + SQLCipher WASM (`nix develop`) +2. Build OpenSSL + SQLCipher WebGL (`nix develop .#webgl`) +3. Run tests and benchmarks +4. Upload `sqlcipher-wasm.zip` and `sqlcipher-webgl.zip` as artifacts +5. Publish to NPM and create GitHub Release on tags ## Troubleshooting @@ -455,7 +453,7 @@ nix develop Rebuild OpenSSL: ```bash -rm -rf openssl-wasm/ openssl-3.3.2/ +rm -rf target/build/ ./build-openssl.sh ``` @@ -463,7 +461,7 @@ rm -rf openssl-wasm/ openssl-3.3.2/ Build first: ```bash -./build.sh +./build-openssl.sh && ./build-wasm.sh ``` ### "file is encrypted or is not a database" @@ -483,7 +481,7 @@ try { Increase Node.js memory: ```bash export NODE_OPTIONS="--max-old-space-size=4096" -./build.sh +./build-wasm.sh ``` ### Cross-platform test fails @@ -539,8 +537,8 @@ Create a GitHub release to trigger automatic publishing to: In the Nix environment: -- `SQLCIPHER_SRC` - Path to SQLCipher source -- `EM_CACHE` - Emscripten cache directory +- `SQLCIPHER_SRC` - Path to SQLCipher source (set by flake.nix) +- `EM_CACHE` - Emscripten cache directory (set by lib.sh per emcc version) - `NODE_PATH` - Node.js module search path ## Resources diff --git a/build-openssl.sh b/build-openssl.sh index e70b1d1..da90ad3 100755 --- a/build-openssl.sh +++ b/build-openssl.sh @@ -1,34 +1,38 @@ #!/usr/bin/env bash set -e +source "$(dirname "$0")/lib.sh" # Build OpenSSL for WebAssembly using Emscripten # This script downloads and compiles OpenSSL 3.x for use with SQLCipher OPENSSL_VERSION="3.3.2" -OPENSSL_DIR="openssl-${OPENSSL_VERSION}" OPENSSL_TAR="openssl-${OPENSSL_VERSION}.tar.gz" -INSTALL_DIR="$(pwd)/openssl-wasm" -echo "Building OpenSSL ${OPENSSL_VERSION} for WASM..." +setup_emscripten -# Download OpenSSL if not already present -if [ ! -d "$OPENSSL_DIR" ]; then - if [ ! -f "$OPENSSL_TAR" ]; then - echo "Downloading OpenSSL..." - wget "https://www.openssl.org/source/${OPENSSL_TAR}" - fi +SRC_DIR="${EMCC_BUILD_DIR}/openssl-${OPENSSL_VERSION}" +INSTALL_DIR="${EMCC_BUILD_DIR}/openssl-wasm" - echo "Extracting OpenSSL..." - tar xzf "$OPENSSL_TAR" +echo "Building OpenSSL ${OPENSSL_VERSION} for WASM (emcc ${EMCC_VERSION})..." + +# Clean previous builds (versioned output only, shared source is reused) +echo "Cleaning previous builds..." +rm -rf "$INSTALL_DIR" + +# Download OpenSSL source (shared across emcc versions) +if [ ! -f "${BUILD_BASE}/$OPENSSL_TAR" ]; then + echo "Downloading OpenSSL..." + wget -P "$BUILD_BASE" "https://www.openssl.org/source/${OPENSSL_TAR}" fi -cd "$OPENSSL_DIR" +# Extract fresh source (configure/make pollute the tree) +rm -rf "$SRC_DIR" +echo "Extracting OpenSSL..." +tar xzf "${BUILD_BASE}/$OPENSSL_TAR" -C "$EMCC_BUILD_DIR" -# Clean previous build -make clean 2>/dev/null || true +pushd "$SRC_DIR" > /dev/null # Configure for WASM -# We need to set CC and other variables for Emscripten echo "Configuring OpenSSL for WASM..." ./Configure \ linux-generic32 \ @@ -44,20 +48,23 @@ echo "Configuring OpenSSL for WASM..." --prefix="$INSTALL_DIR" \ --openssldir="$INSTALL_DIR" \ CC="emcc" \ - CFLAGS="-O3" + AR="emar" \ + RANLIB="emranlib" \ + CFLAGS="-O3 -flto" \ + 2>&1 | tail -20 -# Build +# Build only libraries (skip CLI apps which fail to link under wasm-ld) echo "Building OpenSSL..." -emmake make -j$(nproc) +emmake make -j$(nproc) build_libs 2>&1 | tail -20 -# Install to our prefix +# Install headers and libraries echo "Installing OpenSSL to $INSTALL_DIR..." -make install_sw +mkdir -p "$INSTALL_DIR/lib" "$INSTALL_DIR/include" +cp libcrypto.a libssl.a "$INSTALL_DIR/lib/" +cp -r include/openssl "$INSTALL_DIR/include/" -cd .. +popd > /dev/null echo "" -echo "✓ OpenSSL built successfully!" +echo "OpenSSL built successfully!" echo " Install directory: $INSTALL_DIR" -echo " Headers: $INSTALL_DIR/include" -echo " Libraries: $INSTALL_DIR/lib" diff --git a/build.sh b/build-wasm.sh similarity index 72% rename from build.sh rename to build-wasm.sh index 755e41a..769f37e 100755 --- a/build.sh +++ b/build-wasm.sh @@ -1,57 +1,51 @@ #!/usr/bin/env bash set -e +source "$(dirname "$0")/lib.sh" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color +setup_emscripten +assert_set "$SQLCIPHER_SRC" "SQLCIPHER_SRC not set. Make sure you're in the nix environment." -echo -e "${GREEN}Building SQLCipher for WebAssembly${NC}" +echo -e "${GREEN}Building SQLCipher for WebAssembly (emcc ${EMCC_VERSION})${NC}" echo "========================================" -# Create build directory -BUILD_DIR="build" -DIST_DIR="dist" -mkdir -p "$BUILD_DIR" "$DIST_DIR" +BUILD_DIR="${EMCC_BUILD_DIR}/sqlcipher-wasm" +DIST_DIR="${ROOT_DIR}/dist" +WASM_RESULTS_DIR="${RESULTS_DIR}/sqlcipher-wasm" +rm -rf "$BUILD_DIR" "$DIST_DIR" "$WASM_RESULTS_DIR" "${RESULTS_DIR}/sqlcipher-wasm.zip" +mkdir -p "$BUILD_DIR" "$DIST_DIR" "$WASM_RESULTS_DIR" -# Check if sqlcipher source is available -if [ -z "$SQLCIPHER_SRC" ]; then - echo -e "${RED}Error: SQLCIPHER_SRC not set. Make sure you're in the nix environment.${NC}" - exit 1 -fi +# OpenSSL paths +OPENSSL_ROOT="${EMCC_BUILD_DIR}/openssl-wasm" +OPENSSL_INCLUDE="$OPENSSL_ROOT/include" +OPENSSL_LIB="$OPENSSL_ROOT/lib" # Copy sqlcipher source to build directory echo -e "${YELLOW}Copying SQLCipher source...${NC}" -rsync -a --no-owner --no-group --exclude='.git' "$SQLCIPHER_SRC/" "$BUILD_DIR/sqlcipher/" +rsync -a --no-owner --no-group --exclude='.git' "$SQLCIPHER_SRC/" "$BUILD_DIR/" # Make the build directory writable -chmod -R u+w "$BUILD_DIR/sqlcipher" - -cd "$BUILD_DIR/sqlcipher" +chmod -R u+w "$BUILD_DIR" -# OpenSSL paths -OPENSSL_ROOT="$(cd ../.. && pwd)/openssl-wasm" -OPENSSL_INCLUDE="$OPENSSL_ROOT/include" -OPENSSL_LIB="$OPENSSL_ROOT/lib" +pushd "$BUILD_DIR" > /dev/null # SQLCipher compile flags SQLITE_CFLAGS=( - # Core SQLCipher flags - NOW ENABLED with OpenSSL! + # Core SQLCipher flags "-DSQLITE_HAS_CODEC" "-DSQLCIPHER_CRYPTO_OPENSSL" "-DSQLITE_TEMP_STORE=2" + "-DSQLITE_EXTRA_INIT=sqlcipher_extra_init" + "-DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" # OpenSSL include path "-I$OPENSSL_INCLUDE" # Performance optimizations - "-DSQLITE_THREADSAFE=0" + "-DSQLITE_THREADSAFE=1" "-DSQLITE_OMIT_LOAD_EXTENSION" "-DSQLITE_ENABLE_FTS5" "-DSQLITE_ENABLE_RTREE" - "-DSQLITE_ENABLE_EXPLAIN_COMMENTS" "-DSQLITE_ENABLE_JSON1" # Disable features not available in WASM @@ -98,7 +92,7 @@ EMCC_FLAGS_ESM=( ) echo -e "${YELLOW}Configuring SQLCipher...${NC}" -./configure --with-crypto-lib=none 2>&1 | tail -5 +./configure 2>&1 | tail -5 echo -e "${YELLOW}Creating amalgamation...${NC}" rm -f sqlite3.c sqlite3.h @@ -109,8 +103,7 @@ if [ ! -f sqlite3.c ]; then exit 1 fi -echo -e "${YELLOW}Patching amalgamation...${NC}" -bash ../../patch-amalgamation.sh sqlite3.c +patch_amalgamation sqlite3.c echo -e "${YELLOW}Compiling SQLCipher core...${NC}" emcc "${SQLITE_CFLAGS[@]}" -I. -c sqlite3.c -o sqlite3.o @@ -120,25 +113,22 @@ echo -e "${YELLOW}Building CJS version for Node.js...${NC}" emcc "${SQLITE_CFLAGS[@]}" "${EMCC_FLAGS_CJS[@]}" \ sqlite3.o \ "$OPENSSL_LIB/libcrypto.a" \ - -o "../../$DIST_DIR/sqlcipher.cjs" + -o "$DIST_DIR/sqlcipher.cjs" # Build ESM version for web echo -e "${YELLOW}Building ESM version for web...${NC}" emcc "${SQLITE_CFLAGS[@]}" "${EMCC_FLAGS_ESM[@]}" \ sqlite3.o \ "$OPENSSL_LIB/libcrypto.a" \ - -o "../../$DIST_DIR/sqlcipher.mjs" - -if [ $? -eq 0 ]; then - echo -e "${GREEN}✓ Build successful!${NC}" - echo -e "${GREEN}Output files:${NC}" - ls -lh "../../$DIST_DIR/" -else - echo -e "${RED}✗ Build failed${NC}" - exit 1 -fi + -o "$DIST_DIR/sqlcipher.mjs" + +popd > /dev/null -# Return to root directory -cd ../.. +# Copy to results directory and create archive +cp "$DIST_DIR"/* "$WASM_RESULTS_DIR/" +(cd "$WASM_RESULTS_DIR" && zip -r "${RESULTS_DIR}/sqlcipher-wasm.zip" .) -echo -e "${GREEN}Build complete!${NC}" +echo -e "${GREEN}Build successful!${NC}" +echo -e "${GREEN}Output:${NC}" +ls -lh "$WASM_RESULTS_DIR/" +ls -lh "${RESULTS_DIR}/sqlcipher-wasm.zip" diff --git a/build-webgl.sh b/build-webgl.sh new file mode 100755 index 0000000..0b1b42a --- /dev/null +++ b/build-webgl.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +set -e +source "$(dirname "$0")/lib.sh" + +setup_emscripten +assert_set "$SQLCIPHER_SRC" "SQLCIPHER_SRC not set. Make sure you're in the nix environment (nix develop .#webgl)." + +echo -e "${GREEN}Building SQLCipher .a for Unity WebGL (emcc ${EMCC_VERSION})${NC}" +echo "========================================" + +BUILD_DIR="${EMCC_BUILD_DIR}/sqlcipher-webgl" +WEBGL_RESULTS_DIR="${RESULTS_DIR}/sqlcipher-webgl" +rm -rf "$BUILD_DIR" "$WEBGL_RESULTS_DIR" "${RESULTS_DIR}/sqlcipher-webgl.zip" +mkdir -p "$BUILD_DIR" "$WEBGL_RESULTS_DIR" + +# OpenSSL paths +OPENSSL_ROOT="${EMCC_BUILD_DIR}/openssl-wasm" +OPENSSL_INCLUDE="$OPENSSL_ROOT/include" +OPENSSL_LIB="$OPENSSL_ROOT/lib" + +# Copy sqlcipher source to build directory +echo -e "${YELLOW}Copying SQLCipher source...${NC}" +rsync -a --no-owner --no-group --exclude='.git' "$SQLCIPHER_SRC/" "$BUILD_DIR/" + +# Make the build directory writable +chmod -R u+w "$BUILD_DIR" + +pushd "$BUILD_DIR" > /dev/null + +# SQLCipher compile flags for Unity WebGL +SQLITE_CFLAGS=( + # Core SQLCipher flags + "-DSQLITE_ENABLE_SNAPSHOT" + "-DSQLITE_ENABLE_COLUMN_METADATA" + "-DSQLITE_ENABLE_LOAD_EXTENSION" + "-DSQLITE_ENABLE_API_ARMOR" + "-DSQLITE_ENABLE_FTS3" + "-DSQLITE_ENABLE_FTS3_PARENTHESIS" + "-DSQLITE_ENABLE_FTS4" + "-DSQLITE_ENABLE_FTS5" + "-DSQLITE_ENABLE_MATH_FUNCTIONS" + "-DSQLITE_ENABLE_PREUPDATE_HOOK" + "-DSQLITE_ENABLE_SESSION" + "-DSQLITE_ENABLE_STAT4" + "-DSQLITE_ENABLE_UNLOCK_NOTIFY" + "-DSQLITE_ENABLE_BYTECODE_VTAB" + "-DSQLITE_ENABLE_DBPAGE_VTAB" + "-DSQLITE_ENABLE_DBSTAT_VTAB" + "-DSQLITE_ENABLE_STMTVTAB" + "-DSQLITE_ENABLE_EXPLAIN_COMMENTS" + "-DSQLITE_SOUNDEX" + "-DSQLITE_SYSTEM_MALLOC" + "-DSQLITE_MAX_WORKER_THREADS=8" + "-DSQLITE_THREADSAFE=1" + "-DSQLITE_TEMP_STORE=2" + "-DSQLITE_EXTRA_INIT=sqlcipher_extra_init" + "-DSQLITE_EXTRA_SHUTDOWN=sqlcipher_extra_shutdown" + + # OpenSSL include path + "-DSQLITE_HAS_CODEC" + "-DSQLCIPHER_CRYPTO_OPENSSL" + "-I$OPENSSL_INCLUDE" + + # Performance optimizations + "-DSQLITE_ENABLE_RTREE" + "-DSQLITE_ENABLE_JSON1" + + # Disable features not available in WASM + "-DSQLCIPHER_OMIT_LOG_DEVICE" + + # Memory management + "-DSQLITE_ENABLE_MEMORY_MANAGEMENT" + "-DSQLITE_DEFAULT_MEMSTATUS=0" + + # Optimizations + "-O3" + "-flto" +) + +echo -e "${YELLOW}Configuring SQLCipher...${NC}" +./configure 2>&1 | tail -5 + +echo -e "${YELLOW}Creating amalgamation...${NC}" +rm -f sqlite3.c sqlite3.h +make sqlite3.c 2>&1 | tail -10 + +if [ ! -f sqlite3.c ]; then + echo -e "${RED}Failed to create sqlite3.c amalgamation${NC}" + exit 1 +fi + +patch_amalgamation sqlite3.c + +echo -e "${YELLOW}Compiling SQLCipher core...${NC}" +emcc "${SQLITE_CFLAGS[@]}" -c sqlite3.c -o libsqlcipher.o + +echo -e "${YELLOW}Archiving SQLCipher core...${NC}" +emar rcs libsqlcipher.a libsqlcipher.o + +popd > /dev/null + +# Copy results: SQLCipher + OpenSSL static libs +cp "$BUILD_DIR/libsqlcipher.a" "$WEBGL_RESULTS_DIR/" +mkdir -p "$WEBGL_RESULTS_DIR/openssl" +cp "$OPENSSL_LIB/libcrypto.a" "$OPENSSL_LIB/libssl.a" "$WEBGL_RESULTS_DIR/openssl/" +(cd "$WEBGL_RESULTS_DIR" && zip -r "${RESULTS_DIR}/sqlcipher-webgl.zip" .) + +echo -e "${GREEN}Build successful!${NC}" +echo -e "${GREEN}Output:${NC}" +ls -lh "$WEBGL_RESULTS_DIR/" +ls -lh "${RESULTS_DIR}/sqlcipher-webgl.zip" diff --git a/flake.lock b/flake.lock index f0d2ccf..9116324 100644 --- a/flake.lock +++ b/flake.lock @@ -19,6 +19,22 @@ } }, "nixpkgs": { + "locked": { + "lastModified": 1764521362, + "narHash": "sha256-M101xMtWdF1eSD0xhiR8nG8CXRlHmv6V+VoY65Smwf4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "871b9fd269ff6246794583ce4ee1031e1da71895", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "25.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-emcc-default": { "locked": { "lastModified": 1761907660, "narHash": "sha256-kJ8lIZsiPOmbkJypG+B5sReDXSD1KGu2VEPNqhRa/ew=", @@ -29,15 +45,33 @@ }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", "repo": "nixpkgs", + "rev": "2fb006b87f04c4d3bdf08cfdbc7fab9c13d94a15", + "type": "github" + } + }, + "nixpkgs-emcc-unity": { + "locked": { + "lastModified": 1656250965, + "narHash": "sha256-2IlNf6jxEJiuCrGymqLOLjxk2SIj4HhVIwEb0kvcs24=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a17f325397d137ac4d219ecbd5c7f15154422f4", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a17f325397d137ac4d219ecbd5c7f15154422f4", "type": "github" } }, "root": { "inputs": { "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "nixpkgs-emcc-default": "nixpkgs-emcc-default", + "nixpkgs-emcc-unity": "nixpkgs-emcc-unity" } }, "systems": { diff --git a/flake.nix b/flake.nix index 89a9d74..c7402fc 100644 --- a/flake.nix +++ b/flake.nix @@ -2,105 +2,94 @@ description = "SQLCipher WebAssembly Build"; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/25.11"; flake-utils.url = "github:numtide/flake-utils"; + + # Pinned nixpkgs for emscripten 4.0.12 (working LLVM/wasm-ld with LTO) + nixpkgs-emcc-default.url = "github:NixOS/nixpkgs/2fb006b87f04c4d3bdf08cfdbc7fab9c13d94a15"; + + # Pinned nixpkgs for emscripten 3.1.10 (Unity 2022.3.22f compatible) + nixpkgs-emcc-unity.url = "github:NixOS/nixpkgs/9a17f325397d137ac4d219ecbd5c7f15154422f4"; }; - outputs = { self, nixpkgs, flake-utils }: + outputs = { self, nixpkgs, nixpkgs-emcc-default, nixpkgs-emcc-unity, flake-utils }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + pkgsDefaultEmcc = nixpkgs-emcc-default.legacyPackages.${system}; + pkgsUnityEmcc = nixpkgs-emcc-unity.legacyPackages.${system}; - # SQLCipher source + # SQLCipher v4.9.0 source (shared by both shells) sqlcipherSrc = pkgs.fetchFromGitHub { owner = "sqlcipher"; repo = "sqlcipher"; - rev = "v4.6.1"; - sha256 = "sha256-VcD3NwVrC75kLOJiIgrnzVpkBPhjxTmEFyKg/87wHGc="; + rev = "v4.9.0"; + sha256 = "sha256-FQlTz4iEU4AMzLZNvfYh8IGJHzP3cIXD2NFw8eNMmBU="; }; + commonBuildInputs = with pkgs; [ + gnumake + cmake + gcc + pkg-config + sqlcipher + openssl + nodejs_24 + tcl + git + which + file + coreutils + zip + wget + ]; + + commonShellHook = '' + export SQLCIPHER_SRC="${sqlcipherSrc}" + export NODE_PATH="$PWD/node_modules:$NODE_PATH" + mkdir -p dist target test bench + ''; + in { - devShells.default = pkgs.mkShell { - buildInputs = with pkgs; [ - # Emscripten for WASM compilation - emscripten - - # Build tools - gnumake - cmake - gcc - pkg-config - - # SQLCipher for C++ test program (encrypted SQLite) - sqlcipher - - # OpenSSL and crypto libraries (required by sqlcipher) - openssl - - # Node.js for testing and benchmarking - nodejs_24 - - # TCL for SQLite configuration - tcl - - # Git for fetching sources - git - - # General utilities - which - file - coreutils - ]; - - shellHook = '' - echo "SQLCipher WASM Build Environment" - echo "==================================" - echo "Emscripten version: $(emcc --version | head -n1)" - echo "Node.js version: $(node --version)" - echo "" - echo "Available commands:" - echo " ./build.sh - Build sqlcipher.wasm" - echo " npm test - Run tests" - echo " npm run bench - Run benchmarks" - echo "" - - # Set up environment variables - export SQLCIPHER_SRC="${sqlcipherSrc}" - export EM_CACHE="$PWD/.emscripten-cache" - export NODE_PATH="$PWD/node_modules:$NODE_PATH" - - # Create directories if they don't exist - mkdir -p build dist test bench - ''; - }; - - packages.sqlcipher-wasm = pkgs.stdenv.mkDerivation { - name = "sqlcipher-wasm"; - src = sqlcipherSrc; - - nativeBuildInputs = with pkgs; [ - emscripten - openssl - tcl - ]; - - buildPhase = '' - # Configure for WASM build - emconfigure ./configure \ - --enable-tempstore=yes \ - CFLAGS="-DSQLITE_HAS_CODEC -DSQLITE_TEMP_STORE=2" \ - LDFLAGS="-lcrypto" - - # Build with Emscripten - emmake make - ''; - - installPhase = '' - mkdir -p $out - cp sqlite3.wasm $out/ || true - cp .libs/libsqlcipher.a $out/ || true - ''; + devShells = { + # Default shell: latest emscripten for standalone WASM builds + default = pkgs.mkShell { + buildInputs = [ pkgsDefaultEmcc.emscripten ] ++ commonBuildInputs; + + shellHook = '' + ${commonShellHook} + echo "SQLCipher WASM Build Environment" + echo "==================================" + echo "Emscripten version: $(emcc --version | head -n1)" + echo "Node.js version: $(node --version)" + echo "" + echo "Available commands:" + echo " ./build-openssl.sh - Build OpenSSL for WASM" + echo " ./build-wasm.sh - Build sqlcipher CJS/ESM" + echo " npm test - Run tests" + echo " npm run bench - Run benchmarks" + echo "" + ''; + }; + + # WebGL shell: emscripten 3.1.10 pinned for Unity 2022.3.22f + webgl = pkgs.mkShell { + buildInputs = [ pkgsUnityEmcc.emscripten ] ++ commonBuildInputs; + + shellHook = '' + ${commonShellHook} + echo "SQLCipher WebGL Build Environment" + echo "=========================================" + echo "Emscripten version: $(emcc --version | head -n1)" + echo " (pinned for Unity 2022.3.22f compatibility)" + echo "" + echo "Available commands:" + echo " ./build-openssl.sh - Build OpenSSL for WASM" + echo " ./build-webgl.sh - Build libsqlcipher.a for Unity WebGL" + echo "" + ''; + }; }; } ); diff --git a/lib.sh b/lib.sh new file mode 100644 index 0000000..878d55f --- /dev/null +++ b/lib.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# Shared utilities for sqlcipher-wasm build scripts + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +ROOT_DIR="$(cd "$(dirname "$0")" && pwd)" +BUILD_BASE="${ROOT_DIR}/target/build" +RESULTS_DIR="${ROOT_DIR}/target/results" + +assert_set() { + if [ -z "$1" ]; then + echo -e "${RED}Error: $2${NC}" >&2 + exit 1 + fi +} + +# Detect emscripten version and set up versioned paths: +# EM_CACHE -> .emscripten-cache// +# EMCC_VERSION -> e.g. "4.0.12" +# EMCC_BUILD_DIR -> target/build/emcc- +setup_emscripten() { + EMCC_VERSION="$(emcc --version | head -n1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" + assert_set "$EMCC_VERSION" "Failed to detect emscripten version" + + export EM_CACHE="${ROOT_DIR}/.emscripten-cache/${EMCC_VERSION}" + export EMCC_BUILD_DIR="${BUILD_BASE}/emcc-${EMCC_VERSION}" + + mkdir -p "$EM_CACHE" "$EMCC_BUILD_DIR" "$RESULTS_DIR" +} + +# Patch SQLCipher amalgamation to use OpenSSL provider. +# Disables libtomcrypt/NSS/CommonCrypto providers and removes +# .fini_array usage (unsupported in WASM). +# Usage: patch_amalgamation +patch_amalgamation() { + local sqlite_c="$1" + assert_set "$sqlite_c" "patch_amalgamation: path to sqlite3.c required" + test -f "$sqlite_c" || { echo -e "${RED}Error: $sqlite_c not found${NC}" >&2; exit 1; } + + echo -e "${YELLOW}Patching $sqlite_c for OpenSSL...${NC}" + + # Disable each crypto provider (except OpenSSL) by wrapping in #if 0 + for section in "LIBTOMCRYPT" "NSS" "CC"; do + local label + case "$section" in + LIBTOMCRYPT) label="crypto_libtomcrypt.c" ;; + NSS) label="crypto_nss.c" ;; + CC) label="crypto_cc.c" ;; + esac + + local start end + start=$(grep -n "Begin file $label" "$sqlite_c" | head -1 | cut -d: -f1) + end=$(grep -n "End of $label" "$sqlite_c" | head -1 | cut -d: -f1) + + if [ -n "$start" ] && [ -n "$end" ]; then + echo " Disabling $section provider (lines $start-$end)" + sed "${start}i\\ +#if 0 /* DISABLED - using OpenSSL */ +" "$sqlite_c" > "$sqlite_c.tmp" && mv "$sqlite_c.tmp" "$sqlite_c" + end=$((end + 1)) + sed "${end}a\\ +#endif /* SQLCIPHER_CRYPTO_$section disabled */ +" "$sqlite_c" > "$sqlite_c.tmp" && mv "$sqlite_c.tmp" "$sqlite_c" + fi + done + + # Remove .fini_array usage (unsupported in WASM, crashes LLVM wasm backend) + # Added in SQLCipher 4.9.0 for library cleanup on process exit + echo " Removing .fini_array section attribute (unsupported in WASM)..." + sed 's/__attribute__((used, section(".fini_array")))//' "$sqlite_c" > "$sqlite_c.tmp" && mv "$sqlite_c.tmp" "$sqlite_c" + + echo -e "${GREEN}Patching complete!${NC}" +} diff --git a/package.json b/package.json index 0972d7c..30f25e1 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ }, "scripts": { "prepublishOnly": "npm run build", - "build": "./build.sh", + "build": "./build-wasm.sh", "test": "node test/run-all-tests.cjs", "test:watch": "nodemon --watch test --watch lib --watch dist test/run-all-tests.cjs", "bench": "node bench/benchmark.cjs" diff --git a/patch-amalgamation.sh b/patch-amalgamation.sh deleted file mode 100755 index ff289ba..0000000 --- a/patch-amalgamation.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Patch SQLCipher amalgamation to use OpenSSL provider -# - -set -e - -SQLITE_C="$1" - -if [ ! -f "$SQLITE_C" ]; then - echo "Error: $SQLITE_C not found" - exit 1 -fi - -echo "Patching $SQLITE_C for OpenSSL..." - -# SQLCipher already defaults to OPENSSL, so we just need to disable the other providers -# This prevents compilation errors from unused provider code - -# Find line numbers for each crypto provider section (except OpenSSL) -LIBTOMCRYPT_START=$(grep -n 'Begin file crypto_libtomcrypt.c' "$SQLITE_C" | head -1 | cut -d: -f1) -LIBTOMCRYPT_END=$(grep -n 'End of crypto_libtomcrypt.c' "$SQLITE_C" | head -1 | cut -d: -f1) - -NSS_START=$(grep -n 'Begin file crypto_nss.c' "$SQLITE_C" | head -1 | cut -d: -f1) -NSS_END=$(grep -n 'End of crypto_nss.c' "$SQLITE_C" | head -1 | cut -d: -f1) - -CC_START=$(grep -n 'Begin file crypto_cc.c' "$SQLITE_C" | head -1 | cut -d: -f1) -CC_END=$(grep -n 'End of crypto_cc.c' "$SQLITE_C" | head -1 | cut -d: -f1) - -# Disable each provider (except OpenSSL) by adding #if 0 and #endif -for section in "LIBTOMCRYPT" "NSS" "CC"; do - start_var="${section}_START" - end_var="${section}_END" - start=${!start_var} - end=${!end_var} - - if [ -n "$start" ] && [ -n "$end" ]; then - echo "Disabling $section provider (lines $start-$end)" - # Insert #if 0 before the Begin comment - sed -i "${start}i #if 0 /* DISABLED - using OpenSSL */" "$SQLITE_C" - # Insert #endif after the End comment (add 1 because we just inserted a line) - sed -i "$((end + 1))a #endif /* SQLCIPHER_CRYPTO_$section disabled */" "$SQLITE_C" - fi -done - -echo "Patching complete!"