Postgres Extension

docs/postgres.md

@@ -0,0 +1,82 @@
+# GraphAlg in Postgres
+The goal: Run GraphAlg programs in PostgreSQL.
+This is accomplished by writing a PostgreSQL extension.
+
+## Building and Testing the Extension
+Key steps, explained in more detail below:
+- Build postgreSQL v18 from source and install it to the default path `/usr/local/pgsql`.
+- Setup a dummy database in `~/pgdata`
+- Install SuiteSparse:GraphBLAS
+- Build the extension
+
+
+## Build from source
+Download the latest postgres release source code.
+```bash
+cd thirdparty/
+wget https://ftp.postgresql.org/pub/source/v18.1/postgresql-18.1.tar.bz2
+```
+
+Following https://www.postgresql.org/docs/current/install-make.html
+
+Packages to install:
+- bison
+- flex
+- libreadline-dev
+- libicu-dev
+
+```bash
+./configure
+make
+sudo make install
+```
+
+Now the binaries are located at `/usr/local/pgsql/bin`.
+
+## Setup a database
+```bash
+export LC_ALL=C
+export LC_CTYPE=C
+
+# Create a new DB
+/usr/local/pgsql/bin/initdb -D ~/pgdata
+
+# Start the server
+/usr/local/pgsql/bin/postgres -D ~/pgdata
+
+```
+
+## Load extension
+First build with `pgext/build.sh`.
+
+Then connect to the server with `/usr/local/pgsql/bin/psql postgres` and run:
+
+```bash
+CREATE FUNCTION add_one(integer) RETURNS integer
+     AS '/workspaces/graphalg/pgext/funcs', 'add_one'
+     LANGUAGE C STRICT;
+```
+
+## Foreign Data Wrapper
+Resources:
+
+## GraphBLAS
+We need to link to SuiteSparse:GraphBLAS.
+
+Install package `libgraphblas-dev`.
+
+Alternatively:
+Download https://github.com/DrTimothyAldenDavis/SuiteSparse/archive/refs/tags/v7.12.1.tar.gz to `thirdparty/`.
+
+```
+make
+sudo make install
+```
+
+## Resources
+- https://www.postgresql.org/docs/current/xfunc-c.html
+- https://www.pgedge.com/blog/introduction-to-postgres-extension-development
+- https://stackoverflow.com/questions/76056209/postgresql-c-extension-function-table-as-argument-and-as-result
+- https://www.postgresql.org/docs/current/fdwhandler.html
+- https://www.dolthub.com/blog/2022-01-26-creating-a-postgres-foreign-data-wrapper/
+- https://github.com/Kentik-Archive/wdb_fdw

pg_graphalg/.gitignore

@@ -0,0 +1,2 @@
+/build*
+/.cache

pg_graphalg/CMakeLists.txt

@@ -0,0 +1,36 @@
+# https://cliutils.gitlab.io/modern-cmake/chapters/basics.html#introduction-to-the-basics
+cmake_minimum_required(VERSION 3.15...4.0)
+
+project(
+    pg_graphalg
+    VERSION 0.1
+    DESCRIPTION "GraphAlg extension for PostgreSQL"
+    LANGUAGES C CXX)
+
+find_package(PostgreSQL COMPONENTS Server)
+
+include(FetchContent)
+FetchContent_Declare(
+  GraphAlg
+  SOURCE_DIR ../../compiler
+)
+FetchContent_MakeAvailable(GraphAlg)
+
+# ================================= BEGIN LLVM =================================
+
+find_package(LLVM REQUIRED CONFIG)
+include_directories(${LLVM_INCLUDE_DIRS})
+separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS})
+add_definitions(${LLVM_DEFINITIONS_LIST})
+list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+include(AddLLVM)
+
+llvm_map_components_to_libnames(llvm_libs support)
+
+# ================================== END LLVM ==================================
+
+
+
+find_library(NAMES GraphBLAS)
+
+add_subdirectory(src)

pg_graphalg/README.md

@@ -0,0 +1,67 @@
+# pg_graphalg: Run GraphAlg in PostgreSQL
+This is an extension for PostgreSQL to execute GraphAlg programs.
+
+## Building
+In addition to setting up the devcontainer for the project, you need to:
+- Build postgreSQL v18 from source and install it to the default path `/usr/local/pgsql`.
+- Setup a dummy database in `~/pgdata`
+- Install SuiteSparse:GraphBLAS
+
+### PostgreSQL
+Download the latest postgres release source code.
+
+```bash
+# Dependencies for building PostgreSQL
+sudo apt update
+sudo apt install bison flex libreadline-dev libicu-dev
+
+# Download the sources
+mkdir -p thirdparty/
+cd thirdparty/
+wget https://ftp.postgresql.org/pub/source/v18.1/postgresql-18.1.tar.bz2
+tar xf postgresql-18.1.tar.bz2
+cd postgresql-18.1/
+
+# Build and install
+./configure
+make
+sudo make install
+```
+
+Now PostgreSQL is installed at `/usr/local/pgsql`.
+
+### Setup a database
+```bash
+# Need to set or PostgreSQL will refuse to create the DB
+export LC_ALL=C
+export LC_CTYPE=C
+
+# Create a new DB
+/usr/local/pgsql/bin/initdb -D ~/pgdata
+```
+
+### GraphBLAS
+Install using APT:
+
+```bash
+sudo apt install libgraphblas-dev
+```
+
+### Build the Extension
+```bash
+pg_graphalg/configure.sh
+
+# Extension is located at pg_graphalg/build/src/libpg_graphalg.so
+cmake --build pg_graphalg/build
+```
+
+## Testing
+After the extension has been built, start the server and then run the `test.sql` script:
+
+```bash
+# Start the server
+/usr/local/pgsql/bin/postgres -D ~/pgdata
+
+# Run the tests
+/usr/local/pgsql/bin/psql postgres -f pg_graphalg/test/test.sql
+```

pg_graphalg/configure.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+WORKSPACE_ROOT=pg_graphalg/
+BUILD_DIR=$WORKSPACE_ROOT/build
+rm -rf $BUILD_DIR
+cmake -S $WORKSPACE_ROOT -B $BUILD_DIR -G Ninja \
+    -DCMAKE_BUILD_TYPE=Debug \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+    -DCMAKE_CXX_COMPILER=clang++-20  \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+    -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
+    -DCMAKE_LINKER_TYPE=MOLD \
+    -DPostgreSQL_ROOT=/usr/local/pgsql \
+    -DLLVM_ROOT="/opt/llvm-debug" \

pg_graphalg/include/pg_graphalg/MatrixTable.h

@@ -0,0 +1,92 @@
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <map>
+#include <optional>
+#include <string>
+#include <variant>
+
+namespace pg_graphalg {
+
+enum class MatrixValueType {
+  BOOL,
+  INT,
+  FLOAT,
+};
+
+struct MatrixTableDef {
+  std::string name;
+  std::size_t nRows;
+  std::size_t nCols;
+  MatrixValueType type;
+};
+
+class MatrixTable;
+
+struct MatrixTableScanState {
+  MatrixTable *table;
+  std::size_t row = 0;
+  std::size_t col = 0;
+
+  MatrixTableScanState(MatrixTable *table) : table(table) {}
+
+  void reset() {
+    row = 0;
+    col = 0;
+  }
+};
+
+class MatrixTable {
+private:
+  std::string _name;
+  std::size_t _nRows;
+  std::size_t _nCols;
+  const MatrixValueType _type;
+
+  using AnyValue = std::variant<bool, std::int64_t, double>;
+  std::map<std::pair<std::size_t, std::size_t>, AnyValue> _values;
+
+public:
+  MatrixTable(const MatrixTableDef &def)
+      : _name(def.name), _nRows(def.nRows), _nCols(def.nCols), _type(def.type) {
+  }
+
+  std::size_t nRows() const { return _nRows; }
+  std::size_t nCols() const { return _nCols; }
+  MatrixValueType getType() const { return _type; }
+
+  std::size_t nValues() { return _values.size(); }
+
+  void clear() { _values.clear(); }
+
+  void setValue(std::size_t row, std::size_t col, AnyValue value) {
+    assert(getType() != MatrixValueType::BOOL ||
+           std::holds_alternative<bool>(value));
+    assert(getType() != MatrixValueType::INT ||
+           std::holds_alternative<std::int64_t>(value));
+    assert(getType() != MatrixValueType::FLOAT ||
+           std::holds_alternative<double>(value));
+    _values[{row, col}] = value;
+  }
+
+  const auto &values() const { return _values; }
+
+  std::optional<std::tuple<std::size_t, std::size_t, AnyValue>>
+  scan(MatrixTableScanState &state) {
+    auto it = _values.lower_bound({state.row, state.col});
+    if (it == _values.end()) {
+      return std::nullopt;
+    }
+
+    std::size_t row = it->first.first;
+    std::size_t col = it->first.second;
+    AnyValue val = it->second;
+
+    state.row = row;
+    state.col = col + 1;
+    return std::make_tuple(row, col, val);
+  }
+};
+
+} // namespace pg_graphalg

pg_graphalg/include/pg_graphalg/PgGraphAlg.h

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <memory>
+#include <optional>
+
+#include <llvm/ADT/ArrayRef.h>
+#include <llvm/ADT/DenseMap.h>
+#include <llvm/ADT/StringMap.h>
+#include <llvm/ADT/StringRef.h>
+#include <mlir/IR/Diagnostics.h>
+#include <mlir/IR/DialectRegistry.h>
+#include <mlir/IR/MLIRContext.h>
+#include <mlir/Support/LLVM.h>
+
+#include "pg_graphalg/MatrixTable.h"
+
+namespace pg_graphalg {
+
+using TableId = unsigned int;
+
+class PgGraphAlg {
+private:
+  mlir::DialectRegistry _registry;
+  mlir::MLIRContext _ctx;
+  llvm::DenseMap<TableId, std::unique_ptr<MatrixTable>> _tables;
+
+public:
+  PgGraphAlg(llvm::function_ref<void(mlir::Diagnostic &)> diagHandler);
+
+  std::optional<MatrixTable *> getOrCreateTable(
+      TableId tableId,
+      llvm::function_ref<std::optional<MatrixTableDef>(TableId id)> createFunc);
+
+  bool execute(llvm::StringRef programSource, llvm::StringRef function,
+               llvm::ArrayRef<const MatrixTable *> arguments,
+               MatrixTable &output);
+};
+
+} // namespace pg_graphalg

pg_graphalg/src/CMakeLists.txt

@@ -0,0 +1,16 @@
+add_subdirectory(pg_graphalg)
+
+add_library(pg_graphalg SHARED
+    pg_graphalg.cpp
+)
+target_include_directories(pg_graphalg PUBLIC ../include)
+target_link_libraries(pg_graphalg
+    PRIVATE
+    PgGraphAlg
+    PostgreSQL::PostgreSQL
+    ${llvm_libs}
+    GraphAlgEvaluate
+    GraphAlgIR
+    GraphAlgParse
+    GraphAlgPasses
+)