Merge pull request #637 from swirlai/main_prerelease_2_6

Main prerelease 2 6
swirlai · Sep 26, 2023 · 29569b4 · 29569b4
2 parents f708ece + 80651f5
commit 29569b4
Show file tree

Hide file tree

Showing 25 changed files with 1,035 additions and 146 deletions.
diff --git a/.env.dist b/.env.dist
@@ -11,3 +11,4 @@ SQL_PORT=5432
 MICROSOFT_CLIENT_ID=''
 MICROSOFT_CLIENT_SECRET=''
 MICROSOFT_REDIRECT_URI=''
+OPENAI_API_KEY=
diff --git a/.env.docker b/.env.docker
@@ -11,3 +11,5 @@ SQL_PORT=5432
 MICROSOFT_CLIENT_ID=''
 MICROSOFT_CLIENT_SECRET=''
 MICROSOFT_REDIRECT_URI='http://localhost:8000/swirl/microsoft-callback'
+CELERY_BROKER_URL='redis://redis:6379/0'
+CELERY_RESULT_BACKEND='redis://redis:6379/0'
diff --git a/.github/workflows/integration-api-tests.yml b/.github/workflows/integration-api-tests.yml
@@ -3,7 +3,7 @@ name: IntegrationAPITests
 on:
   workflow_run:
     workflows: [SmokeTests]
-    types: 
+    types:
       - completed
   workflow_dispatch:
 
@@ -14,39 +14,35 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-              python-version: '3.11'
-              cache: 'pip' # caching pip stuff
-      - name: Run Install Swirl
-        run: ./install.sh
-      - name: Update apt
-        run: sudo apt -o Acquire::Retries=3 update
-      - name: upgrade Ubuntu to latest patches
-        run: sudo apt upgrade -y
-      - name: stop update-notifier which restarts datetime
-        run: sudo systemctl stop update-notifier-download.timer    
-      - name: disable update-notifier which restarts datetime
-        run: sudo systemctl disable update-notifier-download.timer
-      - name: Run Install redist-server
-        run: sudo apt install -y redis-server
-      - name: stop system redis
-        run: sudo systemctl stop redis-server.service
-      - name: Set up SWIRL
-        run: python swirl.py setup
-      - name: chown swirl redis.conf
-        run: sudo chown -R redis.redis /home/runner/work/swirl-search/swirl-search/redis.conf
-      - name: Start up SWIRL
-        run: python swirl.py start
-      - name: Run Integrated API tests
-        run: docker run --net=host -t swirlai/swirl-testing:latest-integrated-api sh -c "behave --tags=integrated_api"
-      - name: Upload log files
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: log-files
-          path: |
-            logs/
-            /var/log/syslog*
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+            python-version: '3.11'
+            cache: 'pip' # caching pip stuff
+    - name: Run Install Swirl
+      run: ./install.sh
+    - name: Update apt
+      run: sudo apt -o Acquire::Retries=3 update
+    - name: upgrade Ubuntu to latest patches
+      run: sudo apt upgrade -y
+    - name: stop update-notifier which restarts datetime
+      run: sudo systemctl stop update-notifier-download.timer
+    - name: disable update-notifier which restarts datetime
+      run: sudo systemctl disable update-notifier-download.timer
+    - name: Run Install redist-server
+      run: sudo apt install -y redis-server
+    - name: Set up SWIRL
+      run: python swirl.py setup
+    - name: Start up SWIRL
+      run: python swirl.py start
+    - name: Run Integrated API tests
+      run: docker run --net=host -t swirlai/swirl-testing:latest-integrated-api sh -c "behave --tags=integrated_api"
+    - name: Upload log files
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: log-files
+        path: |
+          logs/
+          /var/log/syslog*
diff --git a/.github/workflows/smoke-tests.yml b/.github/workflows/smoke-tests.yml
@@ -3,7 +3,7 @@ name: SmokeTests
 on:
   workflow_run:
     workflows: [UnitTests]
-    types: 
+    types:
       - completed
   workflow_dispatch:
 
@@ -14,21 +14,31 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-              python-version: '3.10'
-              cache: 'pip' # caching pip stuff
-      - name: Run Install Swirl
-        run: ./install.sh
-      - name: Update apt
-        run: sudo apt -o Acquire::Retries=3 update
-      - name: Run Install redist-server
-        run: sudo apt install -y redis-server
-      - name: Set up SWIRL
-        run: python swirl.py setup
-      - name: Start up SWIRL
-        run: python swirl.py start
-      - name: Run Smoke tests
-        run: docker run --net=host -t swirlai/swirl-testing:latest-smoke-test sh -c "behave **/docker_container/*.feature --tags=docker_api_smoke"
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+            python-version: '3.11'
+            cache: 'pip' # caching pip stuff
+    - name: Run Install Swirl
+      run: ./install.sh
+    - name: Update apt
+      run: sudo apt -o Acquire::Retries=3 update
+    - name: upgrade Ubuntu to latest patches
+      run: sudo apt upgrade -y
+    - name: Run Install redis-server
+      run: sudo apt install -y redis-server
+    - name: Set up SWIRL
+      run: python swirl.py setup
+    - name: Start up SWIRL
+      run: python swirl.py start
+    - name: Run Smoke tests
+      run: docker run --net=host -t swirlai/swirl-testing:latest-smoke-test sh -c "behave **/docker_container/*.feature --tags=docker_api_smoke"
+    - name: Upload log files
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: log-files
+        path: |
+          logs/
+          /var/log/syslog*
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.11.4-slim-bullseye
+FROM python:3.11.5-slim-bookworm
 
 # try to upgrade to a more recent vesion of openssl
 RUN apt-get update

diff --git a/Dockerfile.fork.spg b/Dockerfile.fork.spg
@@ -3,6 +3,8 @@
 FROM node:14.17.5 AS builder
 RUN git clone https://bitbucket.org/swirl-spyglass/spyglass.git /usr/src/spyglass
 WORKDIR /usr/src/spyglass/ui
+RUN git checkout test-sockets
+RUN git pull
 RUN npm install -g npm@7.21.1
 RUN npm install
 RUN npm run build
diff --git a/SearchProviders/europe_pmc.json b/SearchProviders/europe_pmc.json
@@ -6,7 +6,7 @@
     "authenticator": "",
     "connector": "RequestsGet",
     "url": "https://www.ebi.ac.uk/europepmc/webservices/rest/search",
-    "query_template": "{url}?query={query_string}&resultType=core&pageSize=25&format=json",
+    "query_template": "{url}?query={query_string}&resultType=core&format=json",
     "post_query_template": "{}",
     "http_request_headers": {},
     "query_processors": [

diff --git a/SearchProviders/google_news.json b/SearchProviders/google_news.json
@@ -0,0 +1,31 @@
+{
+    "name": "Google News - News.Google.com",
+    "shared": true,
+    "active": false,
+    "default": false,
+    "authenticator": "",
+    "connector": "RequestsGet",
+    "url": "https://news.google.com/rss/search",
+    "query_template": "{url}?q={query_string}",
+    "post_query_template": "{}",
+    "http_request_headers": {},
+    "query_processors": [
+        "AdaptiveQueryProcessor"
+    ],
+    "query_mappings": "",
+    "result_grouping_field": "",
+    "result_processors": [
+        "MappingResultProcessor",
+        "CosineRelevancyResultProcessor"
+    ],
+    "response_mappings": "RESULTS=rss.channel.item",
+    "result_mappings": "title=title,body=description,author=source.['#text'],date_published=pubDate,url=link,source.['@url'],NO_PAYLOAD",
+    "results_per_query": 10,
+    "credentials": "",
+    "eval_credentials": "",
+    "tags": [
+        "GoogleNews",
+        "News",
+        "RSS"
+    ]
+}
diff --git a/SearchProviders/hacker_news.json b/SearchProviders/hacker_news.json
@@ -0,0 +1,65 @@
+[
+    {
+        "name": "Hacker News Stories - HN.Algolia.com",
+        "shared": true,
+        "active": false,
+        "default": false,
+        "authenticator": "",
+        "connector": "RequestsGet",
+        "url": "http://hn.algolia.com/api/v1/search?tags=story",
+        "query_template": "{url}&query={query_string}",
+        "post_query_template": "{}",
+        "http_request_headers": {},
+        "query_processors": [
+            "AdaptiveQueryProcessor"
+        ],
+        "query_mappings": "",
+        "result_grouping_field": "",
+        "result_processors": [
+            "MappingResultProcessor",
+            "CosineRelevancyResultProcessor"
+        ],
+        "response_mappings": "FOUND=nbHits,RESULTS=hits",
+        "result_mappings": "title=title,body=story_text,author=author,date_published=created_at,url='https://news.ycombinator.com/item?id={objectID}',url,points,num_comments,_tags[*],NO_PAYLOAD",
+        "results_per_query": 10,
+        "credentials": "",
+        "eval_credentials": "",
+        "tags": [
+            "HackerNews",
+            "Stories",
+            "Development"
+        ]
+    },
+    {
+        "name": "Hacker News Comments - HN.Algolia.com",
+        "shared": true,
+        "active": false,
+        "default": false,
+        "authenticator": "",
+        "connector": "RequestsGet",
+        "url": "http://hn.algolia.com/api/v1/search?tags=comment",
+        "query_template": "{url}&query={query_string}",
+        "post_query_template": "{}",
+        "http_request_headers": {},
+        "query_processors": [
+            "AdaptiveQueryProcessor"
+        ],
+        "query_mappings": "",
+        "result_grouping_field": "",
+        "result_processors": [
+            "MappingResultProcessor",
+            "CosineRelevancyResultProcessor"
+        ],
+        "response_mappings": "FOUND=nbHits,RESULTS=hits",
+        "result_mappings": "title=story_title,body=comment_text,author=author,date_published=created_at,url='https://news.ycombinator.com/item?id={objectID}',story_url,points,num_comments,_tags[*],NO_PAYLOAD",
+        "results_per_query": 10,
+        "credentials": "",
+        "eval_credentials": "",
+        "tags": [
+            "HackerNews",
+            "Comments",
+            "Development"
+        ]
+    }
+
+]