refactored core from files to folder

commune/__init__.py

@@ -4,9 +4,12 @@
 from .module import Module # the module module
 M = c = Block = Agent  = Module # alias c.Module as c.Block, c.Lego, c.M
 from .vali import Vali # the vali module
-from .server import Server # the server module
-from .client import Client # the client module
+from .server import Server, Client # the server module
 from .key import Key # the key module
 # set the module functions as globalsw
 c.add_to_globals(globals())
+v = Vali 
+s = Server
+k = Key
+# set the module functions as globals
 key = c.get_key # override key function with file key in commune/key.py TODO: remove this line with a better solution

commune/key/__init__.py

@@ -0,0 +1 @@
+from .key import Key

commune/module.py

@@ -42,6 +42,7 @@ class c:
         's' :  'subspace',
         'subspace': 'subspace', 
         'namespace': 'network', 
+        "client": 'server.client',
         'local': 'network',
         'network.local': 'network',
         }
@@ -1594,13 +1595,27 @@ def tree(cls, search=None,  max_age=60,update=False, **kwargs):
         return tree
 
     @classmethod
-    def core_modules(cls, search=None, depth=10000, **kwargs):
+    def core_modules(cls, search=None, depth=10000, avoid_folder_terms = ['modules.'], **kwargs):
         object_paths = cls.find_classes(cls.libpath, depth=depth )
-        object_paths = [cls.objectpath2name(p) for p in object_paths]
+        object_paths = [cls.objectpath2name(p) for p in object_paths if all([avoid not in p for avoid in avoid_folder_terms])]
         if search != None:
             object_paths = [p for p in object_paths if search in p]
         return sorted(list(set(object_paths)))
-
+
+    @classmethod
+    def core_code(cls, search=None, depth=10000, **kwargs):
+        return {k:c.code(k) for k in cls.core_modules(search=search, depth=depth, **kwargs)}
+
+    def core_size(self, search=None, depth=10000, **kwargs):
+        return {k:len(c.code(k)) for k in self.core_modules(search=search, depth=depth, **kwargs)}
+    def module2size(self, search=None, depth=10000, **kwargs):
+        module2size = {}
+        module2code = c.module2code(search=search, depth=depth, **kwargs)
+        for k,v in module2code.items():
+            module2size[k] = len(v)
+        module2size = dict(sorted(module2size.items(), key=lambda x: x[1], reverse=True))
+
+        return module2size
     @classmethod
     def get_modules(cls, search=None, **kwargs):
         return list(cls.tree(search=search, **kwargs).keys())
@@ -1781,6 +1796,20 @@ def module2fns(cls, path=None):
             except Exception as e:
                 pass
         return module2fns
+
+    def module2code(self, search=None, update=False, max_age=60, **kwargs):
+        module2code = {}
+        module2code = c.get('module2code', None, max_age=max_age, update=update)
+        if module2code != None:
+            return module2code
+        module2code = {}
+        for m in c.modules(search=search, **kwargs):
+            try:
+                module2code[m] = c.code(m)
+            except Exception as e:
+                print(f'Error in {m} {e}')
+        c.put('module2code', module2code)
+        return module2code
 
     @classmethod
     def fn2module(cls, path=None):

commune/modules/agent/agent.py

@@ -83,11 +83,11 @@ def process_text(self, text, threshold=1000):
                     'm': c.code,
                     'module': c.code,
                     "run": c.run_fn,
+                    'fn': c.run_fn,
                 }
                 for condition, fn in condition2fn.items():
                     if word.startswith(condition + '/'):
-
-                        word = str(fn(word.split(condition + '/')[-1]))
+                        word = str(fn('/'.join(word.split(condition + '/')[1:])))
                         break
 
             new_text += str(word)

commune/modules/find/find.py

@@ -7,13 +7,15 @@
 import os
 
 class Find:
-    model='anthropic/claude-3.5-sonnet-20240620:beta'
-
     def forward(self,  
-              query='most relevant modules', 
               options: list[str] = [],  
+              query='most relevant', 
+              rules = '''only output the IDX:int  and score OF AT MOST {n}
+                         BUT YOU DONT NEED TO FOR SIMPLICITY''',
+               output_format="DICT(data:list[[idx:int, score:float]])",
               n=10,  
               threshold=0.5,
+              context = None,
               model='anthropic/claude-3.5-sonnet-20240620:beta'):
 
         front_anchor = f"<OUTPUT>"
@@ -28,13 +30,15 @@ def forward(self,
         prompt = f"""
         QUERY
         {query}
+        CONTEXT
+        {context}
         OPTIONS 
         {idx2options} 
-        INSTRUCTION 
-        only output the IDX:int  and score OF AT MOST {n} BUT YOU DONT NEED TO FOR SIMPLICITY
+        RULES 
+        {rules}
         OUTPUT
         (JSON ONLY AND ONLY RESPOND WITH THE FOLLOWING INCLUDING THE ANCHORS SO WE CAN PARSE) 
-        <OUTPUT>DICT(data:list[[idx:int, score:float]])</OUTPUT>
+        <OUTPUT>{output_format}</OUTPUT>
         """
         output = ''
         for ch in c.ask(prompt, model=model): 
@@ -90,8 +94,11 @@ def files(self,
         files =  self.forward(options=files, query=query, n=n, model=model)
         return [c.abspath(path+k) for k in files]
 
-    def modules(self,  query='the filel that is the core of commune', model='anthropic/claude-3.5-sonnet-20240620:beta'): 
-        return self.forward(options=c.get_modules(), query=query, model=model)
+    def modules(self,  query='', model='anthropic/claude-3.5-sonnet-20240620:beta'): 
+        module2fns = []
+        return self.forward(options=c.get_modules(), query=query, model=model, context=c.module2fns())
+
+    def
 
     def utils(self, query='confuse the gradients', model='anthropic/claude-3.5-sonnet-20240620:beta'):
         return self.forward(query=query, options=c.get_utils(), model=model)

commune/network/__init__.py

@@ -0,0 +1,2 @@
+
+from .network import Network

commune/server/__init__.py

@@ -0,0 +1,2 @@
+from .server import Server
+from .client import Client

commune/server.py → commune/server/server.py

@@ -658,7 +658,6 @@ def add_endpoint(self, name, fn):
         assert hasattr(self, name), f'{name} not added to {self.__class__.__name__}'
         return {'success':True, 'message':f'Added {fn} to {self.__class__.__name__}'}
 
-
 
 if __name__ == '__main__':
     Server.run()

commune/vali/README.md

@@ -0,0 +1,64 @@
+```python
+import commune as c
+
+vali = c.module('vali')( network='local', batch_size=128, timeout=3, run_loop=True)
+# Get scoreboard
+scoreboard = vali.scoreboard()
+```
+## Configuration Options
+
+- `network`: Network type ('local', 'bittensor', 'subspace', etc.), to specify the subnet, use {network}/{subnet} (default: 'subspace/storage')
+- `subnet`: Subnet name  (default: None)
+- `batch_size`: Size of parallel processing batches (default: 128)
+- `max_workers`: Number of parallel workers (default: 4)
+- `timeout`: Evaluation timeout in seconds (default: 3)
+- `tempo`: Time between epochs (default: 5)
+- `score`: Score function (Optional)
+- `run_loop`: Enable/disable continuous monitoring
+
+## Key Methods
+
+### score_module
+Evaluates a single module and returns its score.
+
+```python
+result = vali.score_module(module_dict)
+```
+
+### epoch
+Runs a complete evaluation cycle.
+
+```python
+results = vali.epoch()
+```
+
+### scoreboard
+Generates a performance scoreboard.
+
+```python
+board = vali.scoreboard(
+    keys=['name', 'score', 'latency', 'address', 'key'],
+    ascending=True,
+    by='score'
+)
+```
+
+## Testing
+
+```python
+# Run test suite
+Vali.test(
+    n=2,
+    tag='vali_test_net',
+    trials=5,
+    tempo=4
+)
+```
+
+## License
+
+[License Information]
+
+## Contributing
+
+[Contribution Guidelines]

commune/vali/__init__.py

@@ -0,0 +1 @@
+from .vali import Vali # the vali module

setup.py

@@ -22,6 +22,6 @@
     author_email='',
     license='IDGAF License, Do What You Want, I wont sue you',
     install_requires=install_requires,
-    entry_points={'console_scripts': [f'c={libname}.cli:main']},
+    entry_points={'console_scripts': [f'c={libname}.cli.cli:main']},
     classifiers=['FUCK SHIT UP'], 
     python_requires='>=3.8')