Skip to content

Commit 2195352

Browse files
mhordynskimhordynski
authored
Merge branch 'main' into lt/conversation_pipeline_demo2
2 parents 0e2aed6 + 624b6e7 commit 2195352

File tree

282 files changed

+12254
-4497
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

282 files changed

+12254
-4497
lines changed

Diff for: ‎.flake8

-12
This file was deleted.

Diff for: ‎.github/scripts/check_changelog_update.sh

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
echo "Fetching main branch..."
4+
git fetch origin main --depth=1
5+
6+
echo "Identifying changed files between the current branch and main branch..."
7+
CHANGED_FILES=$(git diff --name-only origin/main | tr '\n' ' ')
8+
9+
if [ -z "$CHANGED_FILES" ]; then
10+
echo "No files have been changed in this branch."
11+
exit 0
12+
fi
13+
14+
CHANGED_PACKAGES=$(echo "$CHANGED_FILES" | grep -oE 'packages/[^/]+/src' | cut -d '/' -f2 | sort -u)
15+
16+
if [ -z "$CHANGED_PACKAGES" ]; then
17+
echo "No package changes detected. Skipping changelog check."
18+
exit 0
19+
fi
20+
21+
echo "Found changes in the following packages: $CHANGED_PACKAGES"
22+
23+
# Look for "Changelog-ignore: <package-name>" in the commit message (possibly multiple entries in separate lines)
24+
IGNORED_PACKAGES=$(git log --pretty=format:%B origin/main..HEAD | grep -oP '^Changelog-ignore: \K[^ ]+' | sort -u)
25+
26+
for IGNORED_PACKAGE in $IGNORED_PACKAGES; do
27+
if echo "$CHANGED_PACKAGES" | grep -q "^$IGNORED_PACKAGE$"; then
28+
echo "Ignoring changelog check for package: $IGNORED_PACKAGE"
29+
CHANGED_PACKAGES=$(echo "$CHANGED_PACKAGES" | grep -v "^$IGNORED_PACKAGE$")
30+
fi
31+
done
32+
33+
for PACKAGE in $CHANGED_PACKAGES; do
34+
CHANGELOG="packages/$PACKAGE/CHANGELOG.md"
35+
echo "Checking changelog for package: $PACKAGE"
36+
37+
if ! diff -u <(git show origin/main:$CHANGELOG | grep -Pzo '(?s)(## Unreleased.*?)(?=\n## |\Z)' | tr -d '\0') <(grep -Pzo '(?s)(## Unreleased.*?)(?=\n## |\Z)' $CHANGELOG | tr -d '\0') | grep -q '^\+'; then
38+
echo "No updates detected in changelog for package $PACKAGE. Please add an entry under '## Unreleased'."
39+
exit 1
40+
fi
41+
done
42+
43+
echo "All modified packages have their changelog updates."

Diff for: ‎.github/workflows/ci.yml

+24-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
- name: Install uv
2121
uses: astral-sh/setup-uv@v2
2222
with:
23-
version: "0.4.20"
23+
version: ${{ vars.UV_VERSION || '0.6.9' }}
2424

2525
- name: Set up Python
2626
uses: actions/setup-python@v4
@@ -55,6 +55,7 @@ jobs:
5555
ignore-unfixed: true
5656
exit-code: 0 # change if you want to fail build on vulnerabilities
5757
severity: "CRITICAL,HIGH,MEDIUM"
58+
skip-dirs: .venv,.ruff_cache,.mypy_cache
5859
format: "table"
5960
output: "trivy-scanning-results.txt"
6061

@@ -119,6 +120,28 @@ jobs:
119120
path: dist/**
120121
retention-days: 3
121122

123+
changelog-update-checker:
124+
name: Check Changelog Update
125+
runs-on: ubuntu-latest
126+
steps:
127+
- name: Checkout code
128+
uses: actions/checkout@v4
129+
with:
130+
fetch-depth: 0
131+
132+
- name: Verify changelog updates
133+
run: |
134+
./.github/scripts/check_changelog_update.sh
135+
136+
pr-title-checker:
137+
name: Validate PR title
138+
runs-on: ubuntu-latest
139+
if: github.event_name == 'pull_request'
140+
steps:
141+
- uses: amannn/action-semantic-pull-request@v5
142+
env:
143+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
144+
122145
tests:
123146
name: Run tests
124147
runs-on: ubuntu-latest

Diff for: ‎.github/workflows/documentation.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
- name: Install uv
1919
uses: astral-sh/setup-uv@v2
2020
with:
21-
version: "0.4.20"
21+
version: ${{ vars.UV_VERSION || '0.6.9' }}
2222

2323
- name: Set up Python 3.10
2424
uses: actions/setup-python@v4

Diff for: ‎.github/workflows/prepare_release.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
- name: Install uv
3232
uses: astral-sh/setup-uv@v2
3333
with:
34-
version: "0.4.10"
34+
version: ${{ vars.UV_VERSION || '0.6.9' }}
3535

3636
- name: Set up Python
3737
uses: actions/setup-python@v4

Diff for: ‎.github/workflows/push_release.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
- name: Install uv
1515
uses: astral-sh/setup-uv@v2
1616
with:
17-
version: "0.4.10"
17+
version: ${{ vars.UV_VERSION || '0.6.9' }}
1818

1919
- name: Set up Python
2020
uses: actions/setup-python@v4

Diff for: ‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,5 @@ chroma/
9898
qdrant/
9999

100100
.aider*
101+
102+
.DS_Store

Diff for: ‎.libraries-whitelist.txt

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ chroma-hnswlib
55
rouge
66
distilabel
77
rerankers
8+
py_rust_stemmers

Diff for: ‎README.md

+23-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,28 @@
1414

1515
---
1616

17+
## Features
18+
19+
### 🔨 Build Reliable & Scalable GenAI Apps
20+
- **Swap LLMs anytime** – Switch between [100+ LLMs via LiteLLM](https://ragbits.deepsense.ai/how-to/llms/use_llms/) or run [local models](https://ragbits.deepsense.ai/how-to/llms/use_local_llms/).
21+
- **Type-safe LLM calls** – Use Python generics to [enforce strict type safety](https://ragbits.deepsense.ai/how-to/prompts/use_prompting/#how-to-configure-prompts-output-data-type) in model interactions.
22+
- **Bring your own vector store** – Connect to [Qdrant](https://ragbits.deepsense.ai/api_reference/core/vector-stores/#ragbits.core.vector_stores.qdrant.QdrantVectorStore), [PgVector](https://ragbits.deepsense.ai/api_reference/core/vector-stores/#ragbits.core.vector_stores.pgvector.PgVectorStore), and more with built-in support.
23+
- **Developer tools included** – [Manage vector stores](https://ragbits.deepsense.ai/cli/main/#ragbits-vector-store), query pipelines, and [test prompts from your terminal](https://ragbits.deepsense.ai/quickstart/quickstart1_prompts/#testing-the-prompt-from-the-cli).
24+
- **Modular installation** – Install only what you need, reducing dependencies and improving performance.
25+
26+
### 📚 Fast & Flexible RAG Processing
27+
- **Ingest 20+ formats** – Process PDFs, HTML, spreadsheets, presentations, and more. Process data using [unstructured](https://unstructured.io/) or create a custom provider.
28+
- **Handle complex data** – Extract tables, images, and structured content with built-in VLMs support.
29+
- **Connect to any data source** – Use prebuilt connectors for S3, GCS, Azure, or implement your own.
30+
- **Scale ingestion** – Process large datasets quickly with [Ray-based parallel processing](https://ragbits.deepsense.ai/how-to/document_search/distributed_ingestion/#how-to-ingest-documents-in-a-distributed-fashion).
31+
32+
### 🚀 Deploy & Monitor with Confidence
33+
- **Real-time observability** – Track performance with [OpenTelemetry](https://ragbits.deepsense.ai/how-to/project/use_tracing/#opentelemetry-trace-handler) and [CLI insights](https://ragbits.deepsense.ai/how-to/project/use_tracing/#cli-trace-handler).
34+
- **Built-in testing** – Validate prompts [with promptfoo](https://ragbits.deepsense.ai/how-to/prompts/promptfoo/) before deployment.
35+
- **Auto-optimization** – Continuously evaluate and refine model performance.
36+
- **Visual testing UI (Coming Soon)** – Test and optimize applications with a visual interface.
37+
38+
1739
## What's Included?
1840

1941
- [X] **[Core](https://github.com/deepsense-ai/ragbits/tree/main/packages/ragbits-core)** - Fundamental tools for working with prompts and LLMs.
@@ -78,7 +100,7 @@ print(f"Generated dog name: {response}")
78100
## How Ragbits documentation is organized
79101

80102
- [Quickstart](https://ragbits.deepsense.ai/quickstart/quickstart1_prompts/) - Get started with Ragbits in a few minutes
81-
- [How-to guides](https://ragbits.deepsense.ai/how-to/use_prompting/) - Learn how to use Ragbits in your projects
103+
- [How-to guides](https://ragbits.deepsense.ai/how-to/prompts/use_prompting/) - Learn how to use Ragbits in your projects
82104
- [CLI](https://ragbits.deepsense.ai/cli/main/) - Learn how to run Ragbits in your terminal
83105
- [API reference](https://ragbits.deepsense.ai/api_reference/core/prompt/) - Explore the underlying API of Ragbits
84106

Diff for: ‎docs/api_reference/core/embeddings.md

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
# Embeddings
1+
# Embedders
22

3-
::: ragbits.core.embeddings.Embeddings
3+
::: ragbits.core.embeddings.Embedder
44

5-
::: ragbits.core.embeddings.local.LocalEmbeddings
5+
::: ragbits.core.embeddings.local.LocalEmbedder
66

7-
::: ragbits.core.embeddings.litellm.LiteLLMEmbeddings
7+
::: ragbits.core.embeddings.litellm.LiteLLMEmbedder

Diff for: ‎docs/api_reference/core/vector-stores.md

+4
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,12 @@
66

77
::: ragbits.core.vector_stores.base.VectorStore
88

9+
::: ragbits.core.vector_stores.hybrid.HybridSearchVectorStore
10+
911
::: ragbits.core.vector_stores.in_memory.InMemoryVectorStore
1012

1113
::: ragbits.core.vector_stores.chroma.ChromaVectorStore
1214

1315
::: ragbits.core.vector_stores.qdrant.QdrantVectorStore
16+
17+
::: ragbits.core.vector_stores.pgvector.PgVectorStore

Diff for: ‎docs/api_reference/document_search/documents.md

+17-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,22 @@
44

55
::: ragbits.document_search.documents.document.DocumentType
66

7+
::: ragbits.document_search.documents.document.DocumentMeta
8+
79
::: ragbits.document_search.documents.element.Element
810

9-
::: ragbits.document_search.documents.sources.Source
11+
::: ragbits.document_search.documents.sources.Source
12+
13+
::: ragbits.document_search.documents.sources.AzureBlobStorageSource
14+
15+
::: ragbits.document_search.documents.sources.GCSSource
16+
17+
::: ragbits.document_search.documents.sources.GitSource
18+
19+
::: ragbits.document_search.documents.sources.HuggingFaceSource
20+
21+
::: ragbits.document_search.documents.sources.LocalFileSource
22+
23+
::: ragbits.document_search.documents.sources.S3Source
24+
25+
::: ragbits.document_search.documents.sources.WebSource

Diff for: ‎docs/api_reference/document_search/execution_strategies.md

-7
This file was deleted.

Diff for: ‎docs/api_reference/document_search/ingest/enrichers.md

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Element Enrichers
2+
3+
::: ragbits.document_search.ingestion.enrichers.router.ElementEnricherRouter
4+
5+
::: ragbits.document_search.ingestion.enrichers.base.ElementEnricher
6+
7+
::: ragbits.document_search.ingestion.enrichers.image.ImageElementEnricher

Diff for: ‎docs/api_reference/document_search/ingest/parsers.md

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Document Parsers
2+
3+
::: ragbits.document_search.ingestion.parsers.router.DocumentParserRouter
4+
5+
::: ragbits.document_search.ingestion.parsers.base.DocumentParser
6+
7+
::: ragbits.document_search.ingestion.parsers.base.TextDocumentParser
8+
9+
::: ragbits.document_search.ingestion.parsers.base.ImageDocumentParser
10+
11+
::: ragbits.document_search.ingestion.parsers.unstructured.UnstructuredDocumentParser

Diff for: ‎docs/api_reference/document_search/ingest/strategies.md

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Ingest Strategies
2+
3+
::: ragbits.document_search.ingestion.strategies.base.IngestStrategy
4+
5+
::: ragbits.document_search.ingestion.strategies.sequential.SequentialIngestStrategy
6+
7+
::: ragbits.document_search.ingestion.strategies.batched.BatchedIngestStrategy
8+
9+
::: ragbits.document_search.ingestion.strategies.ray.RayDistributedIngestStrategy

Diff for: ‎docs/api_reference/document_search/processing.md

-3
This file was deleted.

Diff for: ‎docs/api_reference/document_search/retrieval/rephrasers.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Query Rephrasers
2+
3+
::: ragbits.document_search.retrieval.rephrasers.QueryRephraser
4+
::: ragbits.document_search.retrieval.rephrasers.LLMQueryRephraser
5+
::: ragbits.document_search.retrieval.rephrasers.MultiQueryRephraser
6+
::: ragbits.document_search.retrieval.rephrasers.NoopQueryRephraser

Diff for: ‎docs/api_reference/document_search/retrieval/rerankers.md

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Rerankers
2+
3+
::: ragbits.document_search.retrieval.rerankers.base.Reranker
4+
::: ragbits.document_search.retrieval.rerankers.litellm.LiteLLMReranker
5+
::: ragbits.document_search.retrieval.rerankers.noop.NoopReranker

Diff for: ‎docs/api_reference/guardrails/index.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Guardrails
2+
3+
::: ragbits.guardrails.base.Guardrail
4+
::: ragbits.guardrails.base.GuardrailManager
5+
::: ragbits.guardrails.base.GuardrailVerificationResult
6+
::: ragbits.guardrails.openai_moderation.OpenAIModerationGuardrail

Diff for: ‎docs/cli/main.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
# Ragbits CLI
22

3-
Ragbits comes with a command line interface (CLI) that provides a number of commands for working with the Ragbits platform. It can be accessed by running the `ragbits` command in your terminal.
3+
Ragbits comes with a command-line interface (CLI) that provides several commands for working with the Ragbits platform. It can be accessed by running the `ragbits` command in your terminal.
4+
5+
Commands that operate on Ragbits components, such as [`ragbits vector-store`](#ragbits-vector-store), use the project's preferred component implementations if a component configuration is not explicitly provided. To learn how to set component preferences in your project, see the [How-To: Set preferred components in Ragbits project](../how-to/project/component_preferences.md) guide.
46

57
::: mkdocs-click
68
:module: ragbits.cli

Diff for: ‎docs/how-to/document_search/async_processing.md

-73
This file was deleted.

0 commit comments

Comments
 (0)