Keeping the features and Solr architecture we will use.

* Solr6 standalone mode => current production architecture
* Solr8 cloud mode with external zookeeper
* Python application running in docker to manage configsets and collections
* Bash script to index a sample of documents
* Updated the git workflow to build the image used in docker and kubernetes.

.github/workflows/build-manual.yaml

@@ -32,43 +32,27 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ghcr.io/hathitrust/lss-solr:unstable
-
-      - name: Build and push - solr 8, standalone
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./solr8.11.2_files/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/hathitrust/full-text-search-solr:example-8.11
-          target: standalone
 
-      - name: Build and push - solr 8, embedded zookeeper
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./solr8.11.2_files/Dockerfile
-          platforms: linux/amd64,linux/arm64
-          push: true
-          tags: ghcr.io/hathitrust/full-text-search-embedded_zoo:example-8.11
-          target: embedded_zookeeper
-
-      - name: Build and push - solr 8, external zookeeper
+      # use this image to start up a solr cluster in docker-compose
+      -
+        name: Build and push - solr 8, external zookeeper
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./solr8.11.2_files/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ghcr.io/hathitrust/full-text-search-cloud:example-8.11
-          target: external_zookeeper
+          tags: ghcr.io/hathitrust/full-text-search-cloud:shards-docker
+          target: external_zookeeper_docker
 
-      - name: Build and push - solr 8, external zookeeper
+      # use this image to start up a solr cluster in Kubernetes
+      -
+        name: Build and push - solr 8, external zookeeper
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./solr8.11.2_files/Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ghcr.io/hathitrust/full-text-search-cloud:shards-8.11
-          target: external_zookeeper_kubernetes
+          tags: ghcr.io/hathitrust/full-text-search-cloud:shards-kubernetes
+          target: common

docker-compose.yml

@@ -0,0 +1,81 @@
+# a docker-compose.yml to start a simple cluster with 1 ZooKeeper node (external) and 1 Solr nodes.
+# Check this page (https://hathitrust.atlassian.net/wiki/spaces/HAT/pages/3190292502/Solr+cloud+and+external+Zookeeper+parameters)
+# to a better understanding of Solr and Zookeeper set up.
+services:
+  solr_manager:
+    build:
+      context: solr_manager
+      target: runtime
+      dockerfile: Dockerfile
+      args:
+        UID: ${UID:-1000}
+        GID: ${GID:-1000}
+        ENV: ${ENV:-dev}
+        POETRY_VERSION: ${POETRY_VERSION:-1.5.1}
+        SOLR_PASSWORD: ${SOLR_PASSWORD:-solr}
+        SOLR_USER: ${SOLR_USER:-solrRocks}
+        ZK_HOST: ${ZK_HOST:-zoo1:2181}
+    env_file:
+      - solr_manager/.env
+    volumes:
+      - .:/app
+    stdin_open: true
+    depends_on:
+      solr1:
+        condition: service_healthy
+    tty: true
+    container_name: solr_manager
+    networks:
+      - solr
+    profiles: [ solr_collection_manager ]
+  solr1:
+    build:
+      context: .
+      dockerfile: solr8.11.2_cloud/Dockerfile
+      target: external_zookeeper_docker
+    container_name: solr1
+    ports:
+     - "8983:8983"
+    environment:
+      - ZK_HOST=zoo1:2181
+      - SOLR_OPTS=-XX:-UseLargePages
+    networks:
+      - solr
+    depends_on:
+      zoo1:
+        condition: service_healthy
+    volumes:
+      - solr1_data:/var/solr/data
+    command: solr-foreground -c # Solr command to start the container to make sure the security.json is created
+    healthcheck:
+      test: [ "CMD", "/usr/bin/curl", "-s", "-f", "http://solr1:8983/solr/#/admin/ping" ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+  zoo1:
+    image: zookeeper:3.8.0
+    container_name: zoo1
+    restart: always
+    hostname: zoo1
+    ports:
+      - 2181:2181
+      - 7001:7000
+    environment:
+      ZOO_MY_ID: 1
+      ZOO_SERVERS: server.1=zoo1:2888:3888;2181
+      ZOO_4LW_COMMANDS_WHITELIST: mntr, conf, ruok
+      ZOO_CFG_EXTRA: "metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true"
+    networks:
+      - solr
+    volumes:
+      - zoo1_data:/data
+    healthcheck:
+      test: [ "CMD", "echo", "ruok", "|", "nc", "localhost", "2181", "|", "grep", "imok" ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+networks:
+  solr:
+volumes:
+  solr1_data: null
+  zoo1_data: null

docker-compose_solr6_standalone.yml

@@ -1,6 +1,3 @@
----
-version: '3'
-
 services:
   solr-lss-dev:
     image: solr:6.6.6-alpine

indexing_data.sh

@@ -2,12 +2,14 @@
 
 # This script is used to index the sample data into the Solr core
 
-# Example: ./indexing_data.sh http://localhost:8983 ~/mydata data_sample.zip
+# Example: ./indexing_data.sh http://localhost:8983 solr_pass ~/mydata data_sample.zip core-x
 
-solr_url="$1" #Solr URL
+solr_url="$1"
+ #Solr URL
 solr_pass="$2"
 sample_data_directory="$3" #Directory where the sample data is located (XML files)
 zip_file="$4" #Zip file containing the sample data
+collection_name="$5" #Solr collection name
 
 if [ -d "$sample_data_directory" ]
     then
@@ -23,7 +25,7 @@ echo $SOLR_PASS
 for file in "$sample_data_directory/data_sample/"*.xml
           do
               echo "Indexing $file 🌞!!!"
-              curl -u admin:$solr_pass "$solr_url/solr/core-x/update?commit=true" -H "Content-Type: text/xml" --data-binary @$file
+              curl -u admin:$solr_pass "$solr_url/solr/$collection_name/update?commit=true" -H "Content-Type: text/xml" --data-binary @$file
           done
 
 

init_solr_manager.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+if [ -f "solr_manager/.env" ]; then
+  echo "🌎 solr_manager/.env exists. Leaving alone"
+else
+  echo "🌎 solr_manager/.env does not exist. Copying solr_manager/.env-example to solr_manager/.env"
+  cp solr_manager/env.example solr_manager/.env
+
+  YOUR_UID=`id -u`
+  YOUR_GID=`id -g`
+  echo "🙂 Setting your UID ($YOUR_UID) and GID ($YOUR_UID) in .env"
+  docker run --rm -v ./solr_manager/.env:/solr_manager/.env alpine echo "$(sed s/YOUR_UID/$YOUR_UID/ solr_manager/.env)" > solr_manager/.env
+  docker run --rm -v ./solr_manager/.env:/solr_manager/.env alpine echo "$(sed s/YOUR_GID/$YOUR_GID/ solr_manager/.env)" > solr_manager/.env
+fi
+
+echo "🚢 Run containers"
+docker compose -f docker-compose.yml --profile solr_collection_manager up

solr8.11.2_files/Dockerfile → solr8.11.2_cloud/Dockerfile

@@ -13,27 +13,19 @@ ENV SOLR_OPTS="-Denable.packages=true -Dsolr.cloud.client.stallTime=30000 -Dsolr
 -XX:-UseLargePages"
 
 # Copy the required files to the image, lib and conf files
-COPY --chown=solr:solr ./solr8.11.2_files/lib /var/solr/lib
-COPY --chown=solr:solr ./solr8.11.2_files/conf /opt/solr/core-x
+COPY --chown=solr:solr ../solr8.11.2_cloud/lib /var/solr/lib
+COPY --chown=solr:solr ../solr8.11.2_cloud/conf /opt/solr/core-x
 
 
 #=======================================================
 FROM common AS external_zookeeper_docker
 # any steps specific to building the image with external zookeeper
 # authentication is enabled in the solr container
-COPY --chown=solr:solr ./solr8.11.2_files/conf /opt/solr/core-x
+COPY --chown=solr:solr ../solr8.11.2_cloud/security.json /opt/solr/security.json
 
-COPY --chown=solr:solr ./solr8.11.2_files/security.json /opt/solr/security.json
-
-COPY --chown=solr:solr --chmod=0755 ./solr8.11.2_files/solrCloud_external_zooKeeper/init_files/solr_init.sh /usr/bin/solr_init.sh
-
-COPY --chown=solr:solr --chmod=0755 ./solr8.11.2_files/solrCloud_external_zooKeeper/collection_manager.sh /var/solr/data/collection_manager.sh
+COPY --chown=solr:solr --chmod=0755 ../solr8.11.2_cloud/init_files/solr_init.sh /usr/bin/solr_init.sh
 
 # Run this script to start solr as an entrypoint to configure the container to run as an executable
 ENTRYPOINT ["/usr/bin/solr_init.sh"]
 
-#=======================================================
-#FROM common AS external_zookeeper_kubernetes
-# any steps specific to building the image with external zookeeper
-