Merge pull request #300 from SotSF/migration-followups

Migration followups

ONSITE_SETUP.md

@@ -1,5 +1,3 @@
-> NEEDS UPDATE
-
 # Onsite setup
 
 This is the one stop shop for everything related to setting up Conjurer. Just followed these 47 easy steps. What could go wrong?
@@ -19,20 +17,21 @@ It is assumed that there is no internet at the event.
 1. Run `yarn` to install dependencies
 1. Run `yarn dev` to run conjurer locally
 1. Visit http://localhost:3000 to test it out
-   - Log in as someone (top right button), open up an experience (File->Open...)
-   - Click play and verify visuals and audio are working
+   - Note that the first load will be slow
 
-### Setup Conjurer to use local assets
+### Setup Conjurer to use local data
+
+> This section needs a rewrite
 
 These steps can be skipped if you will have internet at the event.
 
 1. Run `yarn downloadCloudAssets` to download all cloud assets into the folder `public/cloud-assets`
    - This could take a little bit because there's a bunch of audio to download
    - Note that you are getting a snapshot of all of the experiences and audio files. If anyone makes more changes to these cloud saved files, you will have to rerun this script to download the latest changed assets
-1. Run `yarn build` (to be confirmed if this is really necessary)
-1. Run `yarn dev` to run conjurer locally if it is not already running
-1. Visit the app at http://localhost:3000
-1. Toggle the `Use local assets` button (desert icon) on such that it becomes orange:
+2. Run `yarn build` (to be confirmed if this is really necessary)
+3. Run `yarn dev` to run conjurer locally if it is not already running
+4. Visit the app at http://localhost:3000
+5. Toggle the `Use local assets` button (desert icon) on such that it becomes orange:
 
 ![Use local assets button](public/use-local-assets-button.png)
 
@@ -47,6 +46,8 @@ You can toggle the same `Use local assets` button again to return to opening/sav
 
 ### Start up and configure Conjurer
 
+> This section needs a rewrite
+
 1. Run `yarn dev` to run Conjurer locally
 1. Visit http://localhost:3000 to tested out
    - Log in as someone, open up an experience
@@ -68,6 +69,7 @@ You can toggle the same `Use local assets` button again to return to opening/sav
 # Optimizations
 
 - Make sure that the conjurer texture size is set to 1024. To the bottom right of the canopy is a button that will say either 256, 512, or 1024. Click it until it says 1024.
+- Make sure the "global intensity" slider (middle right) in conjurer is set to maximum. Use pixlite software for global dimming needs.
 
 # Troubleshooting
 

README.md

@@ -25,39 +25,55 @@ yarn dev
 
 Conjurer should be running locally at http://localhost:3000.
 
-Note that you will be using a local database by default. You will need to define `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` in `.env.local` to access the prod database. Ask for credentials to the Turso dashboard to generate a token.
+### Database access
 
-### Tips
+Note that you will be using a local database by default. The SQLite database will be generated automatically if it does not exist (`setupDatabase.sh`) at `./local.db`.
+
+In production we use [Turso](https://docs.turso.tech/introduction), which cloud hosts SQLite/LibSQL databases. If you would like production database access, you will need to define `TURSO_DATABASE_URL` and `TURSO_AUTH_TOKEN` in `.env.local`:
+
+```
+TURSO_DATABASE_URL=libsql://conjurer-db-secretfire.turso.io
+TURSO_AUTH_TOKEN=******************************
+```
+
+Ask for credentials to the Turso dashboard to generate your own token.
+
+### General info / tips
 
 - In this repo, patterns/effects at their core are just fragment shaders. They may seem scary at first, but with a proper introduction like in [The Book of Shaders](https://thebookofshaders.com/), you too could wield their considerable power!
-- [The shaders page](docs/shaders.md) contains more useful links for learning about shaders.
+- [The shaders page](docs/shaders.md) contains sum useful links for learning about shaders.
 - See the [How to make a pattern page](docs/patterns.md) if you are interested in creating a pattern or effect of your own!
-- We use [Chakra](https://chakra-ui.com/) for our UI in this repo. Check out the [available components here](https://chakra-ui.com/docs/components) as well as the [default theme](https://chakra-ui.com/docs/styled-system/theme)
+- We use [Chakra v2](https://chakra-ui.com/) for our UI in this repo. Check out the [available components here](https://v2.chakra-ui.com/docs/components) as well as the [default theme](https://v2.chakra-ui.com/docs/styled-system/theme)
 - We use [MobX](https://github.com/mobxjs/mobx) for state management. It's not Redux!
-- We use [ThreeJS](https://threejs.org/) and [React Three Fiber](https://docs.pmnd.rs/react-three-fiber/getting-started/introduction) to render the shaders/3D canopy.
-- We use [`react-icons`](https://react-icons.github.io/react-icons/search). Just search for what you want and import the icon from the correct place using the 2-letter prefix.
-- We use [`recharts`](https://recharts.org/en-US/api) to do some simple graphs.
+- We use [ThreeJS](https://threejs.org/) and [React Three Fiber](https://docs.pmnd.rs/react-three-fiber/getting-started/introduction) to render the shaders/3D canopy. In practice, you won't need to understand these libraries unless you are changing something like the rendering pipeline.
+- We use [`react-icons`](https://react-icons.github.io/react-icons/search). Just search for what you want and import the icon from the correct place using the 2-letter prefix (check out other examples in this repo).
+- We use [`recharts`](https://recharts.org/en-US/api) to do some simple graphs for parameter variations.
 - We use [`wavesurfer.js`](https://wavesurfer-js.org/) for all of our audio needs.
+- We use [Turso](https://docs.turso.tech/introduction) to host our production database along with [Drizzle ORM](https://orm.drizzle.team/docs/overview).
+- When running conjurer locally, we instead use a local SQLite database file (`./local.db`). You can run `yarn db:local:studio` to poke around the database via web UI to learn our data models.
+- We use [tRPC](https://trpc.io/docs/), which lets you do remote procedure calls (RPCs) from the client to the server. That means you can just write a javascript function in a router file `/src/server/routers/*Router.ts` and then call it from the client. The rest is abstracted away, so no need to fetch etc.
 
 ### Concepts
 
 - Pattern
-  - A fragment shader that generates a texture (an image) based purely on parameters (uniforms)
+  - A fragment shader that generates a texture (an image) based purely on parameters
   - This texture can either be rendered directly to the canopy or passed to an effect
 - Effect
   - A fragment shader that accepts a texture and applies an effect based purely on parameters, outputting a new texture
   - Just like a pattern, this texture can either be rendered directly to the canopy or passed to an effect
-  - Note: Identical to patterns, except that effects accept a texture as an input
+  - Note: Identical to patterns, except that effects additionally accept a texture as an input
 - Parameter
   - This is a value that tweaks what is being generated by a pattern/effect
+  - A parameter can represent any arbitrary aspect of the pattern that we want
   - "Color", "Fuzziness", "Radius" for example
 - Parameter variations
   - Changes over time applied to a pattern/effect parameter
   - "Change the color from blue to green over 5 seconds"
+  - "Change the wave amplitude from 0 to 1 with an easing function over 1 second"
 
-### Architecture
+### Data flow
 
-Here is the zoomed out view of the architecture. Frame data is sent over websocket to the Unity app's websocket server. Ultimately this data is piped to the canopy and the canopy displays that frame.
+Here is the zoomed out view of the data flow when conjurer is hooked up to the canopy IRL. Frame data is sent over websocket to the Unity app's websocket server. Ultimately this data is piped to the canopy and the canopy displays that frame.
 
 ```mermaid
 graph
@@ -93,10 +109,6 @@ Generates canopy geometry data and stores it in `src/data/canopyGeometry.json`.
 
 Starts a websocket server at port 8080 on localhost. For development use only, to mock the websocket server that the Unity app would run. Writes `src/scripts/output.png` once per second.
 
-#### `yarn downloadCloudAssets`
-
-Downloads all of the experience and audio files from s3 into the folder `public/cloud-assets`. Conjurer can then read from these files when in "local asset mode", useful for situations when internet is not available. See section below for more details.
-
 #### `ANALYZE=true yarn build`
 
 Use webpack analyzer to analyze the bundle. Will launch three tabs in your browser with bundle size details.

package.json

@@ -4,7 +4,7 @@
   "private": true,
   "scripts": {
     "dev": "yarn db:local:setup && next dev",
-    "prod": "src/scripts/try-start-unity-bridge.sh; next dev",
+    "canopy": "src/scripts/try-start-unity-bridge.sh; yarn db:local:setup && next dev",
     "build": "next build",
     "start": "next start",
     "lint": "next lint",
@@ -17,9 +17,7 @@
     "db:prod:download": "src/scripts/downloadDatabase.sh",
     "generateCanopy": "ts-node --project tsconfig.script.json src/scripts/generateCanopy.ts",
     "tursoMigration": "src/scripts/tursoMigration.sh",
-    "downloadCloudAssets": "ts-node --project tsconfig.script.json src/scripts/downloadCloudAssets.ts",
-    "backupLocalExperienceAssets": "ts-node --project tsconfig.script.json src/scripts/backupLocalExperienceAssets.ts",
-    "backup": "ts-node --project tsconfig.script.json src/scripts/backupLocalExperienceAssets.ts",
+    "downloadCloudAudio": "ts-node --project tsconfig.script.json src/scripts/downloadCloudAssets.ts",
     "unityTestServer": "ts-node --project tsconfig.script.json src/scripts/unityTestServer.ts",
     "controllerServer": "ts-node --project tsconfig.script.json src/scripts/controllerServer.ts",
     "generatePattern": "ts-node --project tsconfig.script.json src/scripts/generatePattern.ts"

src/components/BeatMapperPage.tsx

@@ -15,7 +15,7 @@ export const BeatMapperPage = observer(function BeatMapperPage() {
       initialized.current = true;
 
       runInAction(() => {
-        playlistStore.experienceFilenames = ["joe-night-jar"];
+        playlistStore.experienceNames = ["joe-night-jar"];
       });
       store.initializeClientSide();
     }

src/components/PlaylistEditor/PlaylistEditor.tsx

@@ -25,7 +25,7 @@ import { AddExperienceModal } from "@/src/components/PlaylistEditor/AddExperienc
 export const PlaylistEditor = observer(function PlaylistEditor() {
   const store = useStore();
   const { playlistStore, uiStore } = store;
-  const { experienceFilenames } = playlistStore;
+  const { experienceNames } = playlistStore;
 
   const isEditable = store.context !== "viewer";
 
@@ -63,17 +63,16 @@ export const PlaylistEditor = observer(function PlaylistEditor() {
           <Thead>
             <Tr>
               <Th isNumeric>#</Th>
-              <Th>User</Th>
               <Th>Experience name</Th>
             </Tr>
           </Thead>
           <Tbody>
-            {experienceFilenames.map((experienceFilename, index) => (
-              <Tr key={experienceFilename}>
+            {experienceNames.map((experienceName, index) => (
+              <Tr key={experienceName}>
                 <PlaylistItem
-                  experienceFilename={experienceFilename}
+                  experienceName={experienceName}
                   index={index}
-                  playlistLength={experienceFilenames.length}
+                  playlistLength={experienceNames.length}
                   editable={isEditable}
                 />
               </Tr>

src/components/PlaylistEditor/PlaylistItem.tsx

@@ -13,17 +13,16 @@ import { FaPause, FaPlay, FaTrashAlt } from "react-icons/fa";
 import { useState } from "react";
 import { useStore } from "@/src/types/StoreContext";
 import { observer } from "mobx-react-lite";
-import { extractPartsFromExperienceFilename } from "@/src/utils/experience";
 
 type PlaylistItemControlsProps = {
-  experienceFilename: string;
+  experienceName: string;
   index: number;
   playlistLength: number;
   editable?: boolean;
 };
 
 export const PlaylistItem = observer(function PlaylistItem({
-  experienceFilename,
+  experienceName,
   index,
   playlistLength,
   editable,
@@ -35,19 +34,19 @@ export const PlaylistItem = observer(function PlaylistItem({
   const [loadingExperience, setLoadingExperience] = useState(false);
   const onPlayClick = async () => {
     setLoadingExperience(true);
-    await playlistStore.loadAndPlayExperience(experienceFilename);
+    await playlistStore.loadAndPlayExperience(experienceName);
     setLoadingExperience(false);
   };
   const onPauseClick = () => {
     store.pause();
   };
   const onSelect = async () => {
     setLoadingExperience(true);
-    await experienceStore.load(experienceFilename);
+    await experienceStore.load(experienceName);
     setLoadingExperience(false);
   };
 
-  const isSelectedExperience = store.experienceName === experienceFilename;
+  const isSelectedExperience = store.experienceName === experienceName;
 
   const textProps = {
     color: isSelectedExperience ? "blue.400" : undefined,
@@ -57,9 +56,6 @@ export const PlaylistItem = observer(function PlaylistItem({
     onClick: onSelect,
   };
 
-  const { user, experienceName } =
-    extractPartsFromExperienceFilename(experienceFilename);
-
   return (
     <>
       <Td>
@@ -107,10 +103,6 @@ export const PlaylistItem = observer(function PlaylistItem({
         </HStack>
       </Td>
 
-      <Td>
-        <Text {...textProps}>{user}</Text>
-      </Td>
-
       <Td>
         <Text {...textProps}>{experienceName}</Text>
       </Td>

src/data/initialPlaylist.json

@@ -1,19 +1,19 @@
 {
-  "name": "Burning Man event playlist",
-  "experienceFilenames": [
-    "milo-cosmos",
-    "joe-penumbra",
-    "joe-dream-police",
-    "somya-for-the-uncles-and-aunties",
-    "ben-crossing-mars",
-    "ben-concerning-hobbits",
-    "joe-beetleking",
-    "ben-the-blue-whale",
-    "joe-night-jar",
-    "joe-light",
-    "joe-wandersail",
-    "ben-queen-of-all-everything",
-    "justyn-no-skin",
-    "justyn-phantom"
+  "name": "Default playlist",
+  "experienceNames": [
+    "cosmos",
+    "penumbra",
+    "dream-police",
+    "for-the-uncles-and-aunties",
+    "crossing-mars",
+    "concerning-hobbits",
+    "beetleking",
+    "the-blue-whale",
+    "night-jar",
+    "light",
+    "wandersail",
+    "queen-of-all-everything",
+    "no-skin",
+    "phantom"
   ]
 }

src/scripts/assetManagement.ts

@@ -6,7 +6,6 @@ import {
 import {
   ASSET_BUCKET_NAME,
   AUDIO_ASSET_PREFIX,
-  EXPERIENCE_ASSET_PREFIX,
   LOCAL_ASSET_PATH,
 } from "../utils/assets";
 import * as fs from "fs";
@@ -27,41 +26,9 @@ export const saveJson = (filename: string, data: any) =>
 
 export const setupAssetDirectories = () => {
   createDirectory(LOCAL_ASSET_PATH);
-  createDirectory(LOCAL_ASSET_PATH + EXPERIENCE_ASSET_PREFIX);
   createDirectory(LOCAL_ASSET_PATH + AUDIO_ASSET_PREFIX);
 };
 
-export const downloadExperiences = async (s3: S3Client) => {
-  const listObjectsCommand = new ListObjectsCommand({
-    Bucket: ASSET_BUCKET_NAME,
-    Prefix: EXPERIENCE_ASSET_PREFIX,
-  });
-  const data = await s3.send(listObjectsCommand);
-
-  for (const object of data.Contents ?? []) {
-    const filename = object.Key?.split("/")[1] ?? "";
-    const getObjectCommand = new GetObjectCommand({
-      Bucket: ASSET_BUCKET_NAME,
-      Key: `${EXPERIENCE_ASSET_PREFIX}${filename}`,
-    });
-    const data = await s3.send(getObjectCommand);
-    const experienceString = await data.Body?.transformToString();
-    if (experienceString) {
-      const experience = JSON.parse(experienceString);
-      saveJson(
-        `${LOCAL_ASSET_PATH}${EXPERIENCE_ASSET_PREFIX}${filename}`,
-        experience
-      );
-    }
-  }
-};
-
-export const backupExperiences = async () =>
-  await copyDirectory(
-    `${LOCAL_ASSET_PATH}${EXPERIENCE_ASSET_PREFIX}`,
-    `${LOCAL_ASSET_PATH}${Date.now()}_backup_${EXPERIENCE_ASSET_PREFIX}`
-  );
-
 export const downloadAudio = async (s3: S3Client) => {
   const listObjectsCommand = new ListObjectsCommand({
     Bucket: ASSET_BUCKET_NAME,

src/scripts/downloadCloudAudio.ts

@@ -0,0 +1,14 @@
+import { getS3 } from "../utils/s3";
+import { downloadAudio, setupAssetDirectories } from "./assetManagement";
+
+const main = async () => {
+  console.log("creating asset directories as necessary...");
+  setupAssetDirectories();
+
+  const s3 = getS3();
+
+  console.log("downloading audio assets...");
+  await downloadAudio(s3);
+};
+
+main();