diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml deleted file mode 100644 index 0c86d03..0000000 --- a/.github/workflows/pages.yml +++ /dev/null @@ -1,70 +0,0 @@ -# GitHub Pages deployment - Main website with README as homepage -name: Deploy to GitHub Pages - -on: - push: - branches: ["main"] - paths: - - 'README.md' - - 'index.md' - - 'docs/leaderboard.html' - - 'docs/querygym-logo.png' - - '_config.yml' - - '_layouts/**' - - 'Gemfile' - - '.github/workflows/pages.yml' - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -concurrency: - group: "pages" - cancel-in-progress: false - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Pages - uses: actions/configure-pages@v4 - - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: '3.2' - bundler-cache: true - - - name: Install dependencies - run: bundle install - - - name: Build with Jekyll - run: bundle exec jekyll build - env: - JEKYLL_ENV: production - - - name: Copy additional files - run: | - # Copy leaderboard to _site - cp docs/leaderboard.html _site/leaderboard.html - # Copy logo for the site - cp docs/querygym-logo.png _site/querygym-logo.png 2>/dev/null || true - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - needs: build - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/Gemfile b/Gemfile deleted file mode 100644 index 25637eb..0000000 --- a/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -source "https://rubygems.org" - -gem "jekyll", "~> 4.3" -gem "kramdown-parser-gfm" # GitHub Flavored Markdown -gem "webrick" # Required for Ruby 3.0+ diff --git a/README.md b/README.md index a563260..44e5c43 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,10 @@
- ๐ Documentation โข - ๐ Leaderboard โข + ๐ Website โข + ๐ Leaderboard โข + ๐ Dashboard โข + ๐ Docs โข ๐ฆ PyPI โข ๐ Paper
diff --git a/_config.yml b/_config.yml deleted file mode 100644 index cefb360..0000000 --- a/_config.yml +++ /dev/null @@ -1,38 +0,0 @@ -# Jekyll configuration for GitHub Pages -title: QueryGym -description: A lightweight, reproducible toolkit for LLM-based query reformulation -url: "https://ls3-lab.github.io" -baseurl: "/QueryGym" - -# Markdown settings -markdown: kramdown -highlighter: rouge - -# Exclude files from build -exclude: - - Gemfile - - Gemfile.lock - - node_modules - - vendor - - .git - - .github - - tests - - querygym - - examples - - scripts - - "*.py" - - "*.toml" - - Makefile - - Dockerfile - - Dockerfile.cpu - - docker-compose.yml - - .dockerignore - - .gitignore - - requirements*.txt - - docs/ - -# Include specific files -include: - - README.md - - index.md - - LICENSE diff --git a/_layouts/default.html b/_layouts/default.html deleted file mode 100644 index 7809d8f..0000000 --- a/_layouts/default.html +++ /dev/null @@ -1,281 +0,0 @@ - - - - - - -Query Reformulation Methods Performance Evaluation
-- The leaderboard below provides comprehensive performance metrics for QueryGym query reformulation methods - across different LLM configurations and retrieval backends. Each row represents a unique method configuration. - Click on any metric column header (โ ) to sort methods by that metric. Click on any row to expand - and view detailed commands for reproducing the results. -
-| Configuration | -Method | -TREC DL 2019 | -- | TREC DL 2020 | -- | DL HARD | -||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| - | - | AP | -nDCG@10 | -R@1K | -- | AP | -nDCG@10 | -R@1K | -- | AP | -nDCG@10 | -R@1K | -
|
-
-
-
- BM25
-
- k1=0.9, b=0.4
-
-
- |
- - BM25 (Baseline) - | -0.3013 | -0.5058 | -0.7501 | -- | 0.2856 | -0.4796 | -0.7863 | -- | 0.1595 | -0.2850 | -0.6806 | -
|
-
-
-
- GPT-4.1-mini
-
- temp=1.0, max_tokens=128
-
-
-
- BM25
-
- k1=0.9, b=0.4
-
-
- |
- - - Query2Doc (ZS) - | -0.4508 | -0.6709 | -0.8746 | -- | 0.4325 | -0.6323 | -0.8864 | -- | 0.2276 | -0.3303 | -0.7704 | -
| - - Query2Doc (FS) - | -0.4418 | -0.6532 | -0.8521 | -- | 0.4051 | -0.6111 | -0.8869 | -- | 0.2260 | -0.3388 | -0.7842 | -|
| - - Query2Doc (CoT) - | -0.4145 | -0.6128 | -0.8495 | -- | 0.3801 | -0.5894 | -0.8846 | -- | 0.2225 | -0.3191 | -0.7524 | -|
| - - Query2E (ZS) - | -0.3709 | -0.5679 | -0.8384 | -- | 0.3436 | -0.5624 | -0.8373 | -- | 0.1845 | -0.3179 | -0.7642 | -|
| - - CSQE - | -0.4007 | -0.5962 | -0.8506 | -- | 0.3542 | -0.5298 | -0.8431 | -- | 0.2170 | -0.3194 | -0.7276 | -|
| - - LameR - | -0.4185 | -0.6587 | -0.8611 | -- | 0.4432 | -0.6353 | -0.8839 | -- | 0.2562 | -0.3623 | -0.7887 | -|
| - - MuGI - | -0.4766 | -0.6903 | -0.8822 | -- | 0.4353 | -0.6300 | -0.8985 | -- | 0.2393 | -0.3515 | -0.7974 | -|
- All experimental runs shown in the above table can be programmatically executed using the QueryGym pipeline scripts. - The pipeline command runs the complete workflow: reformulation, retrieval, and evaluation. -
- --- --python scripts/querygym_pyserini/pipeline.py \ - --dataset msmarco-v1-passage.dev \ - --method query2doc \ - --model gpt-4 \ - --base-url https://api.openai.com/v1 \ - --api-key YOUR_API_KEY \ - --output-dir runs/query2doc-gpt4-bm25
- For more information, see the QueryGym Pyserini Pipeline documentation. -
-