stable diffusion (#24)

* Add image generation service prompts

* the top-level prompt uses a prompt tool with curl and docker

* typo in the git ref

* used a git prefix instead of a github one (hard to debug currently)

* Add a local tunnel for opening files

* if this is running, this would allow the agent to ask for it to be
  opened

* Add fd to the tool hub

* Build statically linked parser

* Use backend.sock to get token

.gitignore

@@ -2,3 +2,4 @@
 **/result
 .clj-kondo
 .lsp
+/prompts/stable-diffusion/*.png

functions/hub/pull_tools.clj

@@ -6,6 +6,9 @@
    [clojure.string :as string]
    [selmer.parser :as selmer]))
 
+(def new-ones
+  "fd")
+
 (def dumb
 
   "docker-gc"
@@ -64,8 +67,6 @@
    ;; the help menu off stderr
    "hclfmt"
    "fop" ;; xml formatter
-   "dnstracer"
-   "undocker"
    "dockfmt"
 
    ;; just doesn't have any help
@@ -263,6 +264,8 @@
 
 (comment
 
+  (generate "fd")
+
   (doseq [t all-tools]
     (generate t))
 

functions/tree-sitter/.envrc

@@ -0,0 +1 @@
+use flake

functions/tree-sitter/flake.nix

@@ -0,0 +1,91 @@
+{
+  description = "tree-sitter";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = { self, nixpkgs, flake-utils, ...}@inputs:
+
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let
+          pkgs = import nixpkgs {
+            inherit system;
+          };
+
+        in rec
+        {
+          packages = rec {
+
+            # darwin versus linux
+            dylibExt = if nixpkgs.lib.hasInfix "darwin" system then "dylib" else "so";  
+
+            a = pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-python p.tree-sitter-markdown ]);
+
+            markdown-grammar-source = pkgs.fetchFromGitHub {
+              owner = "mdeiml";
+              repo = "tree-sitter-markdown";
+              rev = "28aa3baef73bd458d053b613b8bd10fd102b4405";
+              sha256 = "sha256-HSjKYqjrJKPLbdq1UTvk/KnDqsIzVO7k5syCsIpAZpw=";
+            };
+
+            # build the grammar but does this work for macos?
+            markdown-grammar = (pkgs.tree-sitter.buildGrammar {
+              language = "markdown";
+              version = "0.0.1";
+              src = "${markdown-grammar-source}/tree-sitter-markdown";
+            });
+
+            # derive the parser
+            parser = pkgs.stdenv.mkDerivation {
+              name = "parser";
+              src = ./.;
+              nativeBuildInputs = [ pkgs.gcc
+                                    pkgs.findutils
+                                    pkgs.patchelf ];
+              buildPhase = ''
+                ${pkgs.gcc}/bin/gcc -o parser \
+                  main.c \
+                  -I${markdown-grammar-source}/tree-sitter-markdown/src \
+	          -I${pkgs.tree-sitter}/include \
+                  ${pkgs.tree-sitter}/lib/libtree-sitter.${dylibExt} \
+                  ${markdown-grammar}/parser;
+	      '';
+
+	      installPhase = ''
+	        mkdir -p $out/bin;
+                mkdir -p $out/lib;
+	        cp parser $out/bin/parser;
+                cp ${markdown-grammar}/parser $out/lib/parser;
+	      '';
+
+              fixupPhase = ''
+                find $out -type f -exec patchelf --shrink-rpath '{}' \; -exec strip '{}' \; 2>/dev/null
+              '';
+            };
+
+            # this derivation just contains the init.clj script
+            scripts = pkgs.stdenv.mkDerivation {
+              name = "scripts";
+              src = ./.;
+              installPhase = ''
+                cp init.clj $out
+              '';
+            };
+
+            # the script must have gh in the PATH
+            default = pkgs.writeShellScriptBin "entrypoint" ''
+              export PATH=${pkgs.lib.makeBinPath [parser]}
+              export SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt
+              ${pkgs.babashka}/bin/bb ${scripts} "$@"
+            '';
+          };
+
+          devShells.default = pkgs.mkShell {
+            packages = [ pkgs.tree-sitter pkgs.gcc ];
+          };
+
+        });
+}

functions/tree-sitter/init.clj

@@ -0,0 +1,19 @@
+(ns init
+  (:require
+   [babashka.process]
+   [cheshire.core]))
+
+(try
+  (let [[json-string & args] *command-line-args*
+        {llm-args :args} (cheshire.core/parse-string json-string true)]
+    (println
+     (-> (babashka.process/process
+          {:out :string}
+          (str "curl " llm-args))
+         (deref)
+         (babashka.process/check)
+         :out)))
+  (catch Throwable t
+    (binding [*out* *err*]
+      (println (str "Error: " (.getMessage t)))
+      (System/exit 1))))

functions/tree-sitter/main.c

@@ -0,0 +1,142 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <tree_sitter/api.h>
+#include "tree_sitter/parser.h"
+
+extern const TSLanguage *tree_sitter_markdown(void);
+
+char *escape_double_quotes(const char *type) {
+	char dq[] = "\"";
+	int result = strcmp(type, dq);
+	if (result == 0) {
+		return "\\\"";
+	} else {
+		return type;
+	}
+}
+
+char *escape_back_slash(const char *type) {
+	char dq[] = "\\";
+	int result = strcmp(type, dq);
+	if (result == 0) {
+		return "\\\\";
+	} else {
+		return type;
+	}
+}
+
+// Function to print an S-expression with line and column information
+void print_sexp_with_position(TSNode node, const TSLanguage *lang, const char *source) {
+    TSSymbol symbol = ts_node_symbol(node);
+    const char *type = ts_language_symbol_name( lang, symbol);
+
+    // Get the start position of the node
+    TSPoint start_point = ts_node_start_point(node);
+
+    // Get the end position of the node
+    TSPoint end_point = ts_node_end_point(node);
+
+    char *escaped1 = escape_double_quotes(type);
+    char *escaped = escape_back_slash(escaped1);
+
+    // Print the S-expression with line and column information
+    printf("(\"%s\" \"%d:%d-%d:%d\" ", escaped, start_point.row + 1, start_point.column + 1, end_point.row + 1, end_point.column + 1);
+
+    // Recursively print child nodes
+    for (uint32_t i = 0, child_count = ts_node_child_count(node); i < child_count; i++) {
+        TSNode child = ts_node_child(node, i);
+        print_sexp_with_position(child, lang, source);
+    }
+
+    printf(")");
+}
+
+int main(int argc, char *argv[]) {
+    // Initialize the Tree-sitter library
+    TSParser *parser = ts_parser_new();
+
+    // Load the Markdown language
+    const TSLanguage *markdown_language = tree_sitter_markdown();
+
+    if (!markdown_language) {
+        fprintf(stderr, "Error loading Markdown language\n");
+        return 1;
+    }
+
+    // Set the parser's language to Markdown
+    ts_parser_set_language(parser, markdown_language);
+
+    // Read the Markdown input from a file (change this path to your Markdown file)
+    /*FILE *input_file = fopen(argv[1], "r");*/
+    /*if (!input_file) {*/
+        /*fprintf(stderr, "Error opening input file\n");*/
+        /*return 1;*/
+    /*}*/
+
+    // Get the size of the input file
+    /*fseek(input_file, 0, SEEK_END);*/
+    /*long file_size = ftell(input_file);*/
+    /*rewind(input_file);*/
+
+    // Read the file content into a buffer
+    /*char *input_buffer = (char *)malloc(file_size + 1);*/
+    /*fread(input_buffer, 1, file_size, input_file);*/
+
+    char *input_buffer = NULL;
+    size_t buffer_size = 0;
+    size_t file_size = 0;
+
+    while (1) {
+        char chunk[1024]; // Read data in 1KB chunks
+
+        // Read a chunk of data from stdin
+        size_t bytes_read = fread(chunk, 1, sizeof(chunk), stdin);
+
+        if (bytes_read == 0) {
+            // No more data to read (EOF or error)
+            break;
+        }
+
+        // Resize the buffer to accommodate the new data
+        file_size += bytes_read;
+        input_buffer = (char *)realloc(input_buffer, file_size);
+
+        if (input_buffer == NULL) {
+            perror("Memory allocation failed");
+            exit(EXIT_FAILURE);
+        }
+
+        // Copy the chunk into the buffer
+        memcpy(input_buffer + file_size - bytes_read, chunk, bytes_read);
+    }
+
+    input_buffer[file_size] = '\0';
+
+    // Create a Tree-sitter input document
+    // TSInput input = ts_input_from_string(input_buffer);
+
+    // Parse the input
+    TSTree *tree = ts_parser_parse_string(parser, NULL, input_buffer, file_size);
+
+    TSNode root_node = ts_tree_root_node(tree);
+
+    // Check if parsing was successful
+    if (ts_node_is_null(root_node)) {
+        fprintf(stderr, "Parsing failed\n");
+        return 1;
+    }
+
+    // Perform your desired operations with the parsed tree here
+    // For example, you can traverse the tree and analyze the Markdown document's structure.
+    const char *root_string = ts_node_string(root_node);
+    print_sexp_with_position(root_node, markdown_language, input_buffer);
+
+    // Clean up resources
+    ts_parser_delete(parser);
+    //fclose(input_file);
+    free(input_buffer);
+
+    return 0;
+}
+

functions/tree-sitter/runbook.md

@@ -0,0 +1,14 @@
+## Build
+
+```sh
+nix build .#parser
+```
+
+## Run
+
+The parser will read from stdin and write the sexp to the stdin.
+
+```sh
+./result/bin/parser < <(echo "## hello\n")
+```
+