chore

.github/workflows/docker-publish.yaml

@@ -11,38 +11,29 @@ on:
       - v*
 
   # Run tests for any PRs.
-  pull_request:
+  # pull_request:
 
 env:
   IMAGE_NAME: sync
 
 jobs:
   # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
   push:
     runs-on: ubuntu-latest
     if: github.event_name == 'push'
-
     steps:
       - uses: actions/checkout@v4
 
       - name: Build image
         run: docker build . --file Dockerfile --tag $IMAGE_NAME
 
-      # Use login-action to avoid leaking secrets
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
           registry: docker.pkg.github.com
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      # - name: Log in to Docker Hub
-      #   uses: docker/login-action@v3
-      #   with:
-      #     username: ${{ secrets.DOCKER_USERNAME }}
-      #     password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
       - name: Push image
         run: |
           GH_IMAGE_ID=docker.pkg.github.com/${{ github.repository }}/$IMAGE_NAME

README.md

@@ -10,21 +10,26 @@ For example to create custom metabase dashboards:
 
 The tables currently synced are:
 
-- `matomo_log_visit`
 - `matomo_log_conversion`
+- `matomo_log_visit`
 - `matomo_log_link_visit_action`
 - `matomo_log_action`
 
 ## Usage
 
-Run with docker:
+Adjust your `.env` from the `.env.example` then run :
 
 ```bash
-docker run \
-    -e SOURCE_DATABASE_URL="mysql://user:pass@host:port/database" \
-    -e TARGET_DATABASE_URL="postgresql://user:pass@host:port/database" \
-    -e SITE_ID="1" \
-    ghcr.io/betagouv/matomo-to-pg/sync
+docker run --env-file .env ghcr.io/betagouv/matomo-to-pg/sync
+```
+
+You first need to create your PostgreSQL database structure:
+
+Example with docker:
+
+```
+docker cp ./pg-init.sql [id]:/tmp/pg-init.sql
+docker exec -ti [id] psql --dbname matomo -U matomo -f /tmp/pg-init.sql
 ```
 
 ## Dev

package.json

@@ -1,7 +1,7 @@
 {
   "name": "matomo-to-pg",
   "version": "1.0.0",
-  "main": "src/index.js",
+  "main": "src/index.mjs",
   "license": "Apache-2.0",
   "dependencies": {
     "kysely": "^0.27.4",

src/index.mjs

@@ -2,31 +2,33 @@
 import pg from "pg";
 import { createPool } from "mysql2";
 import { Kysely, PostgresDialect, MysqlDialect } from "kysely";
+import assert from "assert";
 
 const SITE_ID = process.env.SITE_ID;
 const SOURCE_DATABASE_URL = process.env.SOURCE_DATABASE_URL;
 const TARGET_DATABASE_URL = process.env.TARGET_DATABASE_URL;
+const BATCH_ROWS = 500;
 
-const sourceDialect = new MysqlDialect({
-  pool: createPool({
-    uri: SOURCE_DATABASE_URL,
-  }),
-});
-
-const targetDialect = new PostgresDialect({
-  pool: new pg.Pool({
-    connectionString: TARGET_DATABASE_URL,
-  }),
-});
+assert(!!SITE_ID, "process.env.SITE_ID not found");
+assert(!!SOURCE_DATABASE_URL, "process.env.SOURCE_DATABASE_URL not found");
+assert(!!TARGET_DATABASE_URL, "process.env.TARGET_DATABASE_URL not found");
 
 /** @type {Kysely<import("./source").DB>} */
-export const sourceDB = new Kysely({
-  dialect: sourceDialect,
+const sourceDB = new Kysely({
+  dialect: new MysqlDialect({
+    pool: createPool({
+      uri: SOURCE_DATABASE_URL,
+    }),
+  }),
 });
 
 /** @type {Kysely<import("./target").DB>} */
-export const targetDB = new Kysely({
-  dialect: targetDialect,
+const targetDB = new Kysely({
+  dialect: new PostgresDialect({
+    pool: new pg.Pool({
+      connectionString: TARGET_DATABASE_URL,
+    }),
+  }),
 });
 
 /**
@@ -54,8 +56,6 @@ const getMaxValue = async (table, column, idsite = null) => {
   return maxValue;
 };
 
-const BATCH_ROWS = 100;
-
 /**
  *
  * @param {number} idsite
@@ -159,6 +159,8 @@ const importSite = async (idsite) => {
 
   console.log(`copy matomo_log_action from ${lastActionId}...`);
 
+  // we first get a list of possible idaction to optimize the next queries
+  // we do this because matomo_log_action table doesnt have site id
   const validActions = (
     await sourceDB
       .selectFrom("matomo_log_link_visit_action")
@@ -175,7 +177,7 @@ const importSite = async (idsite) => {
     const sourceRows = await sourceDB
       .selectFrom("matomo_log_action")
       .selectAll()
-      .where(({ eb, selectFrom }) => eb("idaction", "in", validActions))
+      .where(({ eb, selectFrom }) => eb("idaction", "in", validActions)) // optim
       .where("idaction", ">", lastActionId)
       .orderBy("idaction")
       .limit(BATCH_ROWS)