Make the database portable and encryptable (#3848)

.gitattributes

@@ -0,0 +1,8 @@
+# Vendored third-party sources. Marking them keeps them out of language
+# statistics and collapses them in diffs, so a routine engine update does not
+# read as a 400,000 line change and does not push review tooling past its
+# size limits.
+vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h linguist-vendored linguist-generated -diff
+vm/ByteCodeTranslator/src/cn1_sqlite3.h             linguist-vendored linguist-generated -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3mc.js              linguist-vendored -diff
+Ports/JavaScriptPort/src/main/webapp/js/sqlite3-opfs-async-proxy.js linguist-vendored -diff

.github/codeql/codeql-config.yml

@@ -9,3 +9,10 @@ name: "Codename One CodeQL configuration"
 # String sinks across the rest of the codebase).
 paths-ignore:
   - 'scripts/cn1playground/**/bsh/cn1/gen/**'
+  # Vendored SQLite engine (SQLite3 Multiple Ciphers), 13MB of upstream C carried
+  # verbatim. Analysing it reports upstream's findings as ours, and its sheer size
+  # pushes CodeQL off incremental analysis so it re-reports pre-existing alerts
+  # across unrelated files. Provenance and update instructions are in
+  # vm/ByteCodeTranslator/src/cn1_sqlite3_README.md.
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3_amalgamation.h'
+  - 'vm/ByteCodeTranslator/src/cn1_sqlite3.h'

.github/workflows/linux-build-run.yml

@@ -88,9 +88,14 @@ jobs:
         with: { distribution: 'temurin', java-version: '8' }
 
       - name: Build plugin + cn1-ads-mock (JDK 8)
+        # Maven Central intermittently serves 403/429 from the runner CDN edge and Maven treats it
+        # as a permanent resolution failure, which killed this job at dependency download before
+        # anything was compiled. See scripts/ci/retry.sh; these steps only build, so a blanket
+        # retry cannot turn a flaky test into a pass.
         run: |
           cd maven
-          JAVA_HOME="$JDK_8_HOME" mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
+          JAVA_HOME="$JDK_8_HOME" bash ../scripts/ci/retry.sh \
+            mvn -B -pl codenameone-maven-plugin,cn1-ads-mock -am \
             -DskipTests '-Dmaven.javadoc.skip=true' '-Plocal-dev-javase' install
 
       - name: Build hellocodenameone-common (JDK 17)
@@ -102,7 +107,8 @@ jobs:
           sleep 2
           mkdir -p "$HOME/.codenameone"
           [ -f "$HOME/.codenameone/guibuilder.jar" ] || : > "$HOME/.codenameone/guibuilder.jar"
-          mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
+          bash scripts/ci/retry.sh \
+            mvn -B -f scripts/hellocodenameone/pom.xml -pl common -am install -DskipTests '-Dmaven.javadoc.skip=true'
 
       - name: Package suite classes
         run: |

.github/workflows/pr.yml

@@ -181,6 +181,15 @@ jobs:
             -Dcodename1.platform=javase \
             -Dcn1.binaries="${CN1_BINARIES}" \
             -pl javase -Plocal-dev-javase test
+      - name: Verify encrypted databases interoperate with reference SQLCipher
+        if: ${{ matrix.java-version == 8 }}
+        # The portability claim is that every port writes one on-disk format. A cipher
+        # misconfiguration produces files each platform reads happily and nothing else can
+        # touch, which no single-platform test would catch, so check ours against the stock
+        # sqlcipher client in both directions.
+        run: |
+          apt-get update -qq && apt-get install -y -qq sqlcipher
+          scripts/ci/db-cipher-interop.sh
       - name: Determine AI/ad cn1libs to build
         id: cn1libs
         if: ${{ matrix.java-version == 8 }}
@@ -344,6 +353,20 @@ jobs:
             -Dcn1.binaries="${CN1_BINARIES}" \
             -Pcompile-android \
             -pl android,ios,codenameone-maven-plugin -am verify
+      - name: Type-check the SQLCipher-backed database package
+        if: ${{ matrix.java-version == 8 }}
+        env:
+          CN1_BINARIES: ${{ github.workspace }}/maven/target/cn1-binaries
+        # That package is excluded from the port build on purpose, so nothing above compiles it
+        # and a member of com.codename1.impl.android it reaches can stop being public without
+        # anything noticing until an Android app build forty minutes later. Runs after the port
+        # is compiled, which is what it type-checks against.
+        run: scripts/ci/check-android-cipher-package-compiles.sh
+      - name: Compile the bundled SQLite WebAssembly module
+        if: ${{ matrix.java-version == 8 }}
+        # A module no engine accepts does not fail anything on its own: the JavaScript port just
+        # reports the database as unavailable and its tests skip.
+        run: scripts/ci/check-js-sqlite-wasm.sh
       - name: Run SpotBugs for ByteCodeTranslator
         if: ${{ matrix.java-version == 8 }}
         run: mvn -B -DskipTests=true -f vm/ByteCodeTranslator/pom.xml verify

CodenameOne/src/com/codename1/db/Cursor.java

@@ -24,124 +24,117 @@
 
 import java.io.IOException;
 
-/// The Cursor interface is used to iterate over the results returned from a database query.
-/// **IMPORTANT:** Notice that some methods might not be supported on all platforms!
+/// Iterates over the results returned from a database query.
 ///
-/// There is more thorough coverage of the `Database API here`.
-///
-/// The sample code below presents a Database Explorer tool that allows executing arbitrary SQL and
-/// viewing the tabular results:
+/// Positions are counted from zero and a new cursor sits before the first row, so the usual loop
+/// is simply:
 ///
 /// ```java
-/// Toolbar.setGlobalToolbar(true);
-/// Style s = UIManager.getInstance().getComponentStyle("TitleCommand");
-/// FontImage icon = FontImage.createMaterial(FontImage.MATERIAL_QUERY_BUILDER, s);
-/// Form hi = new Form("SQL Explorer", new BorderLayout());
-/// hi.getToolbar().addCommandToRightBar("", icon, (e) -> {
-///     TextArea query = new TextArea(3, 80);
-///     Command ok = new Command("Execute");
-///     Command cancel = new Command("Cancel");
-///     if(Dialog.show("Query", query, ok, cancel) == ok) {
-///         Database db = null;
-///         Cursor cur = null;
-///         try {
-///             db = Display.getInstance().openOrCreate("MyDB.db");
-///             if(query.getText().startsWith("select")) {
-///                 cur = db.executeQuery(query.getText());
-///                 int columns = cur.getColumnCount();
-///                 hi.removeAll();
-///                 if(columns > 0) {
-///                     boolean next = cur.next();
-///                     if(next) {
-///                         ArrayList data = new ArrayList<>();
-///                         String[] columnNames = new String[columns];
-///                         for(int iter = 0 ; iter
+/// Cursor cur = db.executeQuery("SELECT id, body FROM notes ORDER BY id");
+/// try {
+///     while (cur.next()) {
+///         Row row = cur.getRow();
+///         System.out.println(row.getInteger(0) + ": " + row.getString(1));
+///     }
+/// } finally {
+///     cur.close();
+/// }
+/// ```
+///
+/// Every navigation method works on every platform. Only the cost varies: `#next()` is uniformly
+/// cheap, while `#last()`, `#prev()` and `#position(int)` may have to rewind and re-step the
+/// underlying statement, which costs time proportional to the distance from the start. For a large
+/// result set, prefer iterating forward with `#next()`.
+///
+/// Because a backward seek re-runs the statement, a cursor is a repeatable read only inside a
+/// transaction. See the `com.codename1.db` package documentation for the full contract.
 ///
 /// @author Chen
 public interface Cursor {
 
-    /// Move the cursor to the first row.
-    ///
-    /// If cursor provides forward-only navigation and is positioned after the
-    /// first row then calling first() method would throw a IOException.
+    /// Moves the cursor onto the first row.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if there is a first row, false for an empty result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     // PMD Fix (UnnecessaryModifier): Interface methods are implicitly public; remove redundant modifiers.
     boolean first() throws IOException;
 
-    /// Move the cursor to the last row.
+    /// Moves the cursor onto the last row.
+    ///
+    /// Costs a full pass over the result set the first time it is called.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if there is a last row, false for an empty result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean last() throws IOException;
 
-    /// Moves the cursor to the next row.
-    /// Calling next() method the first time will position cursor on the first.
+    /// Advances the cursor one row.
+    ///
+    /// A new cursor sits before the first row, so the first call lands on it.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if a row was reached, false at the end of the result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean next() throws IOException;
 
-    /// Moves the cursor to the previous row.
-    /// If cursor is forward type then calling prev() would throw a IOException.
+    /// Moves the cursor back one row.
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if a row was reached, false when already at or before the first row
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean prev() throws IOException;
 
-    /// Returns the zero-based index for a given column name.
-    /// Note that columns meta information is available only after navigation to
-    /// the first row
+    /// Returns the zero-based index of a column, or -1 if there is no such column.
+    ///
+    /// The comparison is case-insensitive and matches the result set label, so a column selected
+    /// as `SELECT a AS b` is found under `b`. Available as soon as the query returns, before the
+    /// first `#next()`.
     ///
     /// #### Parameters
     ///
-    /// - `columnName`: the name of the column.
+    /// - `columnName`: the name of the column
     ///
     /// #### Returns
     ///
-    /// the index of the column
+    /// the zero-based index, or -1 when the column is not in the result set
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     int getColumnIndex(String columnName) throws IOException;
 
-    /// Returns the column name at a given zero-based column index.
-    /// Note that columns meta information is available only after navigation to
-    /// the first row
+    /// Returns the label of the column at a zero-based index.
+    ///
+    /// Available as soon as the query returns, before the first `#next()`.
     ///
     /// #### Parameters
     ///
-    /// - `columnIndex`: the index of the column
+    /// - `columnIndex`: the zero-based index of the column
     ///
     /// #### Returns
     ///
-    /// the name of the column
+    /// the column label
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     String getColumnName(int columnIndex) throws IOException;
 
     /// Returns the column count
@@ -155,48 +148,57 @@ public interface Cursor {
     /// - `IOException`
     int getColumnCount() throws IOException;
 
-    /// Returns the current Cursor position.
+    /// Returns the zero-based position of the cursor.
+    ///
+    /// Reports -1 before any successful move, and the row count once the result set is exhausted.
     ///
     /// #### Returns
     ///
     /// the cursor position
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     int getPosition() throws IOException;
 
-    /// Move the cursor to an absolute row position
+    /// Moves the cursor to an absolute zero-based row.
+    ///
+    /// Passing -1 rewinds to before the first row and returns false.
     ///
     /// #### Parameters
     ///
-    /// - `row`: position to move to
+    /// - `row`: the zero-based row to move to
     ///
     /// #### Returns
     ///
-    /// true if succeeded
+    /// true if the row exists, false if it is out of range
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the cursor is closed
     boolean position(int row) throws IOException;
 
-    /// Close the cursor and release its resources
+    /// Closes the cursor and releases its resources.
+    ///
+    /// Calling this more than once is harmless.
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: if the underlying statement cannot be released
     void close() throws IOException;
 
-    /// Get the Row data Object.
+    /// Returns the current row.
+    ///
+    /// Valid only while the cursor is on a row.
     ///
     /// #### Returns
     ///
-    /// a Row Object
+    /// the current row
     ///
     /// #### Throws
     ///
-    /// - `IOException`
+    /// - `IOException`: @throws IOException if the cursor is closed, or is before the first row or
+    ///                   past the last one
     Row getRow() throws IOException;
 
 }