Init...

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,65 @@
+name: Bug
+description: Report a problem with AIDE
+title: '[Bug]: '
+labels: ['bug']
+body:
+  - type: markdown
+    attributes:
+      value: Thank you for taking the time to fill out this bug report. Please provide as much information as possible to help us understand and address the issue effectively.
+
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for the same bug?
+      description: Please check if an issue already exists for the bug you encountered.
+      options:
+      - label: I have checked the existing issues.
+        required: true
+
+  - type: textarea
+    id: bug-description
+    attributes:
+      label: Describe the bug and reproduction steps
+      description: Provide a description of the issue along with any reproduction steps.
+    validations:
+      required: true
+
+  - type: dropdown
+    id: installation
+    attributes:
+      label: AIDE Installation
+      description: How are you running AIDE?
+      options:
+        - Commandline
+        - Docker
+        - Python Script
+        - Other
+      default: 0
+
+  - type: input
+    id: aideml-version
+    attributes:
+      label: AIDE Version
+      description: What version of AIDE are you using?
+      placeholder: ex. 0.1.4, main, etc.
+
+  - type: input
+    id: llm
+    attributes:
+      label: LLM
+      description: Which llm are you using?
+      placeholder: ex. gpt-4-turbo, claude-3-5-sonnet, etc.
+
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating System
+      options:
+        - MacOS
+        - Linux
+        - WSL on Windows
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Logs, Errors, Screenshots, and Additional Context
+      description: Please provide any additional information you think might help.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature Request
+about: Suggest an idea for new AIDE features
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+**What problem or use case are you trying to solve?**
+
+**Describe the UX of the solution you'd like**
+
+**Do you have thoughts on the technical implementation?**
+
+**Describe alternatives you've considered**
+
+**Additional context**

.github/ISSUE_TEMPLATE/technical_proposal.md

@@ -0,0 +1,18 @@
+---
+name: Technical Proposal
+about: Propose a new architecture or technology
+title: ''
+labels: 'proposal'
+assignees: ''
+
+---
+
+**Summary**
+
+**Motivation**
+
+**Technical Design**
+
+**Alternatives to Consider**
+
+**Additional context**

.github/pull_request_template.md

@@ -0,0 +1,8 @@
+## Description
+
+* a simple description of what you're trying to accomplish
+* a summary of changes in code
+* which issues it fixes, if any
+
+## Screenshots/videos:
+

.github/workflows/linter.yml

@@ -0,0 +1,25 @@
+name: Linter
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  lint-python:
+    name: Python Linting
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install Dependencies
+        run: |
+          pip install ruff==0.7.1
+          pip install black==24.3.0
+      - name: Run Ruff
+        run: ruff check --output-format=github aide/
+      - name: Run Black
+        run: black --check --diff aide/

.github/workflows/python-publish.yml

@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish on PyPI
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -0,0 +1,172 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+.vscode/
+
+workspaces
+logs
+
+.DS_STORE
+.trunk
+
+.gradio/
+.ruff_cache/
+
+.env

.python-version

@@ -0,0 +1 @@
+3.10

.streamlit/config.toml

@@ -0,0 +1,17 @@
+[theme]
+# Primary colors
+primaryColor="#0D0F18"  # --wecopink: 343 98% 63%
+backgroundColor="#F0EFE9" # --background: 49 10% 94%
+secondaryBackgroundColor="#FFFFFF" # --card: 60 33.3% 98%
+textColor="#0A0A0A" # --primary: 0 0% 17%
+
+# Font
+font="sans serif"
+
+[ui]
+hideTopBar = true
+
+[client]
+toolbarMode = "minimal"
+showErrorDetails = true
+showSidebarNavigation = false

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Weco AI Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

aide/__init__.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+
+from .agent import Agent
+from .interpreter import Interpreter
+from .journal import Journal
+from omegaconf import OmegaConf
+from rich.status import Status
+from .utils.config import (
+    load_task_desc,
+    prep_agent_workspace,
+    save_run,
+    _load_cfg,
+    prep_cfg,
+)
+
+
+@dataclass
+class Solution:
+    code: str
+    valid_metric: float
+
+
+class Experiment:
+
+    def __init__(self, data_dir: str, goal: str, eval: str | None = None):
+        """Initialize a new experiment run.
+
+        Args:
+            data_dir (str): Path to the directory containing the data files.
+            goal (str): Description of the goal of the task.
+            eval (str | None, optional): Optional description of the preferred way for the agent to evaluate its solutions.
+        """
+
+        _cfg = _load_cfg(use_cli_args=False)
+        _cfg.data_dir = data_dir
+        _cfg.goal = goal
+        _cfg.eval = eval
+        self.cfg = prep_cfg(_cfg)
+
+        self.task_desc = load_task_desc(self.cfg)
+
+        with Status("Preparing agent workspace (copying and extracting files) ..."):
+            prep_agent_workspace(self.cfg)
+
+        self.journal = Journal()
+        self.agent = Agent(
+            task_desc=self.task_desc,
+            cfg=self.cfg,
+            journal=self.journal,
+        )
+        self.interpreter = Interpreter(
+            self.cfg.workspace_dir, **OmegaConf.to_container(self.cfg.exec)  # type: ignore
+        )
+
+    def run(self, steps: int) -> Solution:
+        for _i in range(steps):
+            self.agent.step(exec_callback=self.interpreter.run)
+            save_run(self.cfg, self.journal)
+        self.interpreter.cleanup_session()
+
+        best_node = self.journal.get_best_node(only_good=False)
+        return Solution(code=best_node.code, valid_metric=best_node.metric.value)

aide/agent.py

@@ -0,0 +1,339 @@
+import logging
+import random
+from typing import Any, Callable, cast
+
+import humanize
+from .backend import FunctionSpec, query
+from .interpreter import ExecutionResult
+from .journal import Journal, Node
+from .utils import data_preview
+from .utils.config import Config
+from .utils.metric import MetricValue, WorstMetricValue
+from .utils.response import extract_code, extract_text_up_to_code, wrap_code
+
+logger = logging.getLogger("aide")
+
+
+ExecCallbackType = Callable[[str, bool], ExecutionResult]
+
+review_func_spec = FunctionSpec(
+    name="submit_review",
+    json_schema={
+        "type": "object",
+        "properties": {
+            "is_bug": {
+                "type": "boolean",
+                "description": "true if the output log shows that the execution failed or has some bug, otherwise false.",
+            },
+            "summary": {
+                "type": "string",
+                "description": "if there is a bug, propose a fix. Otherwise, write a short summary (2-3 sentences) describing the empirical findings.",
+            },
+            "metric": {
+                "type": "number",
+                "description": "If the code ran successfully, report the value of the validation metric. Otherwise, leave it null.",
+            },
+            "lower_is_better": {
+                "type": "boolean",
+                "description": "true if the metric should be minimized (i.e. a lower metric value is better, such as with MSE), false if the metric should be maximized (i.e. a higher metric value is better, such as with accuracy).",
+            },
+        },
+        "required": ["is_bug", "summary", "metric", "lower_is_better"],
+    },
+    description="Submit a review evaluating the output of the training script.",
+)
+
+
+class Agent:
+    def __init__(
+        self,
+        task_desc: str,
+        cfg: Config,
+        journal: Journal,
+    ):
+        super().__init__()
+        self.task_desc = task_desc
+        self.cfg = cfg
+        self.acfg = cfg.agent
+        self.journal = journal
+        self.data_preview: str | None = None
+
+    def search_policy(self) -> Node | None:
+        """Select a node to work on (or None to draft a new node)."""
+        search_cfg = self.acfg.search
+
+        # initial drafting
+        if len(self.journal.draft_nodes) < search_cfg.num_drafts:
+            logger.debug("[search policy] drafting new node (not enough drafts)")
+            return None
+
+        # debugging
+        if random.random() < search_cfg.debug_prob:
+            # nodes that are buggy + leaf nodes + debug depth < max debug depth
+            debuggable_nodes = [
+                n
+                for n in self.journal.buggy_nodes
+                if (n.is_leaf and n.debug_depth <= search_cfg.max_debug_depth)
+            ]
+            if debuggable_nodes:
+                logger.debug("[search policy] debugging")
+                return random.choice(debuggable_nodes)
+            logger.debug("[search policy] not debugging by chance")
+
+        # back to drafting if no nodes to improve
+        good_nodes = self.journal.good_nodes
+        if not good_nodes:
+            logger.debug("[search policy] drafting new node (no good nodes)")
+            return None
+
+        # greedy
+        greedy_node = self.journal.get_best_node()
+        logger.debug("[search policy] greedy node selected")
+        return greedy_node
+
+    @property
+    def _prompt_environment(self):
+        pkgs = [
+            "numpy",
+            "pandas",
+            "scikit-learn",
+            "statsmodels",
+            "xgboost",
+            "lightGBM",
+            "torch",
+            "torchvision",
+            "torch-geometric",
+            "bayesian-optimization",
+            "timm",
+        ]
+        random.shuffle(pkgs)
+        pkg_str = ", ".join([f"`{p}`" for p in pkgs])
+
+        env_prompt = {
+            "Installed Packages": f"Your solution can use any relevant machine learning packages such as: {pkg_str}. Feel free to use any other packages too (all packages are already installed!). For neural networks we suggest using PyTorch rather than TensorFlow."
+        }
+        return env_prompt
+
+    @property
+    def _prompt_impl_guideline(self):
+        impl_guideline = [
+            "The code should **implement the proposed solution** and **print the value of the evaluation metric computed on a hold-out validation set**.",
+            "The code should be a single-file python program that is self-contained and can be executed as-is.",
+            "No parts of the code should be skipped, don't terminate the before finishing the script.",
+            "Your response should only contain a single code block.",
+            f"Be aware of the running time of the code, it should complete within {humanize.naturaldelta(self.cfg.exec.timeout)}.",
+            'All the provided input data is stored in "./input" directory.',
+            '**If there is test data provided for this task, please save the test predictions in a `submission.csv` file in the "./working" directory as described in the task description** This is extremely important since this file is used for grading/evaluation. DO NOT FORGET THE submission.csv file!',
+            'You can also use the "./working" directory to store any temporary files that your code needs to create.',
+        ]
+        if self.acfg.expose_prediction:
+            impl_guideline.append(
+                "The implementation should include a predict() function, "
+                "allowing users to seamlessly reuse the code to make predictions on new data. "
+                "The prediction function should be well-documented, especially the function signature."
+            )
+
+        if self.acfg.k_fold_validation > 1:
+            impl_guideline.append(
+                f"The evaluation should be based on {self.acfg.k_fold_validation}-fold cross-validation but only if that's an appropriate evaluation for the task at hand."
+            )
+
+        return {"Implementation guideline": impl_guideline}
+
+    @property
+    def _prompt_resp_fmt(self):
+        return {
+            "Response format": (
+                "Your response should be a brief outline/sketch of your proposed solution in natural language (3-5 sentences), "
+                "followed by a single markdown code block (wrapped in ```) which implements this solution and prints out the evaluation metric. "
+                "There should be no additional headings or text in your response. Just natural language text followed by a newline and then the markdown code block. "
+            )
+        }
+
+    def plan_and_code_query(self, prompt, retries=3) -> tuple[str, str]:
+        """Generate a natural language plan + code in the same LLM call and split them apart."""
+        completion_text = None
+        for _ in range(retries):
+            completion_text = query(
+                system_message=prompt,
+                user_message=None,
+                model=self.acfg.code.model,
+                temperature=self.acfg.code.temp,
+            )
+
+            code = extract_code(completion_text)
+            nl_text = extract_text_up_to_code(completion_text)
+
+            if code and nl_text:
+                # merge all code blocks into a single string
+                return nl_text, code
+
+            print("Plan + code extraction failed, retrying...")
+        print("Final plan + code extraction attempt failed, giving up...")
+        return "", completion_text  # type: ignore
+
+    def _draft(self) -> Node:
+        prompt: Any = {
+            "Introduction": (
+                "You are a Kaggle grandmaster attending a competition. "
+                "In order to win this competition, you need to come up with an excellent and creative plan "
+                "for a solution and then implement this solution in Python. We will now provide a description of the task."
+            ),
+            "Task description": self.task_desc,
+            "Memory": self.journal.generate_summary(),
+            "Instructions": {},
+        }
+        prompt["Instructions"] |= self._prompt_resp_fmt
+        prompt["Instructions"] |= {
+            "Solution sketch guideline": [
+                "This first solution design should be relatively simple, without ensembling or hyper-parameter optimization.",
+                "Take the Memory section into consideration when proposing the design,"
+                " don't propose the same modelling solution but keep the evaluation the same.",
+                "The solution sketch should be 3-5 sentences.",
+                "Propose an evaluation metric that is reasonable for this task.",
+                "Don't suggest to do EDA.",
+                "The data is already prepared and available in the `./input` directory. There is no need to unzip any files.",
+            ],
+        }
+        prompt["Instructions"] |= self._prompt_impl_guideline
+        prompt["Instructions"] |= self._prompt_environment
+
+        if self.acfg.data_preview:
+            prompt["Data Overview"] = self.data_preview
+
+        plan, code = self.plan_and_code_query(prompt)
+        return Node(plan=plan, code=code)
+
+    def _improve(self, parent_node: Node) -> Node:
+        prompt: Any = {
+            "Introduction": (
+                "You are a Kaggle grandmaster attending a competition. You are provided with a previously developed "
+                "solution below and should improve it in order to further increase the (test time) performance. "
+                "For this you should first outline a brief plan in natural language for how the solution can be improved and "
+                "then implement this improvement in Python based on the provided previous solution. "
+            ),
+            "Task description": self.task_desc,
+            "Memory": self.journal.generate_summary(),
+            "Instructions": {},
+        }
+        prompt["Previous solution"] = {
+            "Code": wrap_code(parent_node.code),
+        }
+
+        prompt["Instructions"] |= self._prompt_resp_fmt
+        prompt["Instructions"] |= {
+            "Solution improvement sketch guideline": [
+                "The solution sketch should be a brief natural language description of how the previous solution can be improved.",
+                "You should be very specific and should only propose a single actionable improvement.",
+                "This improvement should be atomic so that we can experimentally evaluate the effect of the proposed change.",
+                "Take the Memory section into consideration when proposing the improvement.",
+                "The solution sketch should be 3-5 sentences.",
+                "Don't suggest to do EDA.",
+            ],
+        }
+        prompt["Instructions"] |= self._prompt_impl_guideline
+
+        plan, code = self.plan_and_code_query(prompt)
+        return Node(
+            plan=plan,
+            code=code,
+            parent=parent_node,
+        )
+
+    def _debug(self, parent_node: Node) -> Node:
+        prompt: Any = {
+            "Introduction": (
+                "You are a Kaggle grandmaster attending a competition. "
+                "Your previous solution had a bug, so based on the information below, you should revise it in order to fix this bug. "
+                "Your response should be an implementation outline in natural language,"
+                " followed by a single markdown code block which implements the bugfix/solution."
+            ),
+            "Task description": self.task_desc,
+            "Previous (buggy) implementation": wrap_code(parent_node.code),
+            "Execution output": wrap_code(parent_node.term_out, lang=""),
+            "Instructions": {},
+        }
+        prompt["Instructions"] |= self._prompt_resp_fmt
+        prompt["Instructions"] |= {
+            "Bugfix improvement sketch guideline": [
+                "You should write a brief natural language description (3-5 sentences) of how the issue in the previous implementation can be fixed.",
+                "Don't suggest to do EDA.",
+            ],
+        }
+        prompt["Instructions"] |= self._prompt_impl_guideline
+
+        if self.acfg.data_preview:
+            prompt["Data Overview"] = self.data_preview
+
+        plan, code = self.plan_and_code_query(prompt)
+        return Node(plan=plan, code=code, parent=parent_node)
+
+    def update_data_preview(
+        self,
+    ):
+        self.data_preview = data_preview.generate(self.cfg.workspace_dir)
+
+    def step(self, exec_callback: ExecCallbackType):
+        if not self.journal.nodes or self.data_preview is None:
+            self.update_data_preview()
+
+        parent_node = self.search_policy()
+        logger.debug(f"Agent is generating code, parent node type: {type(parent_node)}")
+
+        if parent_node is None:
+            result_node = self._draft()
+        elif parent_node.is_buggy:
+            result_node = self._debug(parent_node)
+        else:
+            result_node = self._improve(parent_node)
+
+        self.parse_exec_result(
+            node=result_node,
+            exec_result=exec_callback(result_node.code, True),
+        )
+        self.journal.append(result_node)
+
+    def parse_exec_result(self, node: Node, exec_result: ExecutionResult):
+        logger.info(f"Agent is parsing execution results for node {node.id}")
+
+        node.absorb_exec_result(exec_result)
+
+        prompt = {
+            "Introduction": (
+                "You are a Kaggle grandmaster attending a competition. "
+                "You have written code to solve this task and now need to evaluate the output of the code execution. "
+                "You should determine if there were any bugs as well as report the empirical findings."
+            ),
+            "Task description": self.task_desc,
+            "Implementation": wrap_code(node.code),
+            "Execution output": wrap_code(node.term_out, lang=""),
+        }
+
+        response = cast(
+            dict,
+            query(
+                system_message=prompt,
+                user_message=None,
+                func_spec=review_func_spec,
+                model=self.acfg.feedback.model,
+                temperature=self.acfg.feedback.temp,
+            ),
+        )
+
+        # if the metric isn't a float then fill the metric with the worst metric
+        if not isinstance(response["metric"], float):
+            response["metric"] = None
+
+        node.analysis = response["summary"]
+        node.is_buggy = (
+            response["is_bug"]
+            or node.exc_type is not None
+            or response["metric"] is None
+        )
+
+        if node.is_buggy:
+            node.metric = WorstMetricValue()
+        else:
+            node.metric = MetricValue(
+                response["metric"], maximize=not response["lower_is_better"]
+            )

aide/backend/__init__.py

@@ -0,0 +1,52 @@
+from . import backend_anthropic, backend_openai
+from .utils import FunctionSpec, OutputType, PromptType, compile_prompt_to_md
+
+
+def query(
+    system_message: PromptType | None,
+    user_message: PromptType | None,
+    model: str,
+    temperature: float | None = None,
+    max_tokens: int | None = None,
+    func_spec: FunctionSpec | None = None,
+    **model_kwargs,
+) -> OutputType:
+    """
+    General LLM query for various backends with a single system and user message.
+    Supports function calling for some backends.
+
+    Args:
+        system_message (PromptType | None): Uncompiled system message (will generate a message following the OpenAI/Anthropic format)
+        user_message (PromptType | None): Uncompiled user message (will generate a message following the OpenAI/Anthropic format)
+        model (str): string identifier for the model to use (e.g. "gpt-4-turbo")
+        temperature (float | None, optional): Temperature to sample at. Defaults to the model-specific default.
+        max_tokens (int | None, optional): Maximum number of tokens to generate. Defaults to the model-specific max tokens.
+        func_spec (FunctionSpec | None, optional): Optional FunctionSpec object defining a function call. If given, the return value will be a dict.
+
+    Returns:
+        OutputType: A string completion if func_spec is None, otherwise a dict with the function call details.
+    """
+
+    model_kwargs = model_kwargs | {
+        "model": model,
+        "temperature": temperature,
+        "max_tokens": max_tokens,
+    }
+
+    # Handle models with beta limitations
+    # ref: https://platform.openai.com/docs/guides/reasoning/beta-limitations
+    if model.startswith("o1"):
+        if system_message:
+            user_message = system_message
+        system_message = None
+        model_kwargs["temperature"] = 1
+
+    query_func = backend_anthropic.query if "claude-" in model else backend_openai.query
+    output, req_time, in_tok_count, out_tok_count, info = query_func(
+        system_message=compile_prompt_to_md(system_message) if system_message else None,
+        user_message=compile_prompt_to_md(user_message) if user_message else None,
+        func_spec=func_spec,
+        **model_kwargs,
+    )
+
+    return output

aide/backend/backend_anthropic.py

@@ -0,0 +1,107 @@
+"""Backend for Anthropic API."""
+
+import logging
+import time
+
+from .utils import FunctionSpec, OutputType, opt_messages_to_list, backoff_create
+from funcy import notnone, once, select_values
+import anthropic
+
+logger = logging.getLogger("aide")
+
+_client: anthropic.Anthropic = None  # type: ignore
+
+ANTHROPIC_TIMEOUT_EXCEPTIONS = (
+    anthropic.RateLimitError,
+    anthropic.APIConnectionError,
+    anthropic.APITimeoutError,
+    anthropic.InternalServerError,
+)
+
+ANTHROPIC_MODEL_ALIASES = {
+    "claude-3.5-sonnet": "claude-3-sonnet-20241022",
+}
+
+
+@once
+def _setup_anthropic_client():
+    global _client
+    _client = anthropic.Anthropic(max_retries=0)
+
+
+def query(
+    system_message: str | None,
+    user_message: str | None,
+    func_spec: FunctionSpec | None = None,
+    **model_kwargs,
+) -> tuple[OutputType, float, int, int, dict]:
+    """
+    Query Anthropic's API, optionally with tool use (Anthropic's equivalent to function calling).
+    """
+    _setup_anthropic_client()
+
+    filtered_kwargs: dict = select_values(notnone, model_kwargs)  # type: ignore
+    if "max_tokens" not in filtered_kwargs:
+        filtered_kwargs["max_tokens"] = 4096  # default for Claude models
+
+    model_name = filtered_kwargs.get("model", "")
+    logger.debug(f"Anthropic query called with model='{model_name}'")
+
+    if model_name in ANTHROPIC_MODEL_ALIASES:
+        model_name = ANTHROPIC_MODEL_ALIASES[model_name]
+
+    if func_spec is not None and func_spec.name == "submit_review":
+        filtered_kwargs["tools"] = [func_spec.as_anthropic_tool_dict]
+        # Force tool use
+        filtered_kwargs["tool_choice"] = func_spec.anthropic_tool_choice_dict
+
+    # Anthropic doesn't allow not having user messages
+    # if we only have system msg -> use it as user msg
+    if system_message is not None and user_message is None:
+        system_message, user_message = user_message, system_message
+
+    # Anthropic passes system messages as a separate argument
+    if system_message is not None:
+        filtered_kwargs["system"] = system_message
+
+    messages = opt_messages_to_list(None, user_message)
+
+    t0 = time.time()
+    message = backoff_create(
+        _client.messages.create,
+        ANTHROPIC_TIMEOUT_EXCEPTIONS,
+        messages=messages,
+        **filtered_kwargs,
+    )
+    req_time = time.time() - t0
+
+    # Handle tool calls if present
+    if (
+        func_spec is not None
+        and "tools" in filtered_kwargs
+        and len(message.content) > 0
+        and message.content[0].type == "tool_use"
+    ):
+        block = message.content[0]  # This is a "ToolUseBlock"
+        # block has attributes: type, id, name, input
+        assert (
+            block.name == func_spec.name
+        ), f"Function name mismatch: expected {func_spec.name}, got {block.name}"
+        output = block.input  # Anthropic calls the parameters "input"
+    else:
+        # For non-tool responses, ensure we have text content
+        assert len(message.content) == 1, "Expected single content item"
+        assert (
+            message.content[0].type == "text"
+        ), f"Expected text response, got {message.content[0].type}"
+        output = message.content[0].text
+
+    in_tokens = message.usage.input_tokens
+    out_tokens = message.usage.output_tokens
+
+    info = {
+        "stop_reason": message.stop_reason,
+        "model": message.model,
+    }
+
+    return output, req_time, in_tokens, out_tokens, info

aide/backend/backend_openai.py

@@ -0,0 +1,127 @@
+"""Backend for OpenAI API."""
+
+import json
+import logging
+import time
+
+from .utils import FunctionSpec, OutputType, opt_messages_to_list, backoff_create
+from funcy import notnone, once, select_values
+import openai
+
+logger = logging.getLogger("aide")
+
+_client: openai.OpenAI = None  # type: ignore
+
+OPENAI_TIMEOUT_EXCEPTIONS = (
+    openai.RateLimitError,
+    openai.APIConnectionError,
+    openai.APITimeoutError,
+    openai.InternalServerError,
+)
+
+
+@once
+def _setup_openai_client():
+    global _client
+    _client = openai.OpenAI(max_retries=0)
+
+
+def query(
+    system_message: str | None,
+    user_message: str | None,
+    func_spec: FunctionSpec | None = None,
+    **model_kwargs,
+) -> tuple[OutputType, float, int, int, dict]:
+    """
+    Query the OpenAI API, optionally with function calling.
+    If the model doesn't support function calling, gracefully degrade to text generation.
+    """
+    _setup_openai_client()
+    filtered_kwargs: dict = select_values(notnone, model_kwargs)
+
+    # Convert system/user messages to the format required by the client
+    messages = opt_messages_to_list(system_message, user_message)
+
+    # If function calling is requested, attach the function spec
+    if func_spec is not None:
+        filtered_kwargs["tools"] = [func_spec.as_openai_tool_dict]
+        filtered_kwargs["tool_choice"] = func_spec.openai_tool_choice_dict
+
+    completion = None
+    t0 = time.time()
+
+    # Attempt the API call
+    try:
+        completion = backoff_create(
+            _client.chat.completions.create,
+            OPENAI_TIMEOUT_EXCEPTIONS,
+            messages=messages,
+            **filtered_kwargs,
+        )
+    except openai.error.BadRequestError as e:
+        # Check whether the error indicates that function calling is not supported
+        if "function calling" in str(e).lower() or "tools" in str(e).lower():
+            logger.warning(
+                "Function calling was attempted but is not supported by this model. "
+                "Falling back to plain text generation."
+            )
+            # Remove function-calling parameters and retry
+            filtered_kwargs.pop("tools", None)
+            filtered_kwargs.pop("tool_choice", None)
+
+            # Retry without function calling
+            completion = backoff_create(
+                _client.chat.completions.create,
+                OPENAI_TIMEOUT_EXCEPTIONS,
+                messages=messages,
+                **filtered_kwargs,
+            )
+        else:
+            # If it's some other error, re-raise
+            raise
+
+    req_time = time.time() - t0
+    choice = completion.choices[0]
+
+    # Decide how to parse the response
+    if func_spec is None or "tools" not in filtered_kwargs:
+        # No function calling was ultimately used
+        output = choice.message.content
+    else:
+        # Attempt to extract tool calls
+        tool_calls = getattr(choice.message, "tool_calls", None)
+        if not tool_calls:
+            logger.warning(
+                "No function call was used despite function spec. Fallback to text.\n"
+                f"Message content: {choice.message.content}"
+            )
+            output = choice.message.content
+        else:
+            first_call = tool_calls[0]
+            # Optional: verify that the function name matches
+            if first_call.function.name != func_spec.name:
+                logger.warning(
+                    f"Function name mismatch: expected {func_spec.name}, "
+                    f"got {first_call.function.name}. Fallback to text."
+                )
+                output = choice.message.content
+            else:
+                try:
+                    output = json.loads(first_call.function.arguments)
+                except json.JSONDecodeError as ex:
+                    logger.error(
+                        "Error decoding function arguments:\n"
+                        f"{first_call.function.arguments}"
+                    )
+                    raise ex
+
+    in_tokens = completion.usage.prompt_tokens
+    out_tokens = completion.usage.completion_tokens
+
+    info = {
+        "system_fingerprint": completion.system_fingerprint,
+        "model": completion.model,
+        "created": completion.created,
+    }
+
+    return output, req_time, in_tokens, out_tokens, info

aide/backend/utils.py

@@ -0,0 +1,101 @@
+from dataclasses import dataclass
+
+import jsonschema
+from dataclasses_json import DataClassJsonMixin
+import backoff
+import logging
+from typing import Callable
+
+PromptType = str | dict | list
+FunctionCallType = dict
+OutputType = str | FunctionCallType
+
+
+logger = logging.getLogger("aide")
+
+
+@backoff.on_predicate(
+    wait_gen=backoff.expo,
+    max_value=60,
+    factor=1.5,
+)
+def backoff_create(
+    create_fn: Callable, retry_exceptions: list[Exception], *args, **kwargs
+):
+    try:
+        return create_fn(*args, **kwargs)
+    except retry_exceptions as e:
+        logger.info(f"Backoff exception: {e}")
+        return False
+
+
+def opt_messages_to_list(
+    system_message: str | None, user_message: str | None
+) -> list[dict[str, str]]:
+    messages = []
+    if system_message:
+        messages.append({"role": "system", "content": system_message})
+    if user_message:
+        messages.append({"role": "user", "content": user_message})
+    return messages
+
+
+def compile_prompt_to_md(prompt: PromptType, _header_depth: int = 1) -> str:
+    if isinstance(prompt, str):
+        return prompt.strip() + "\n"
+    elif isinstance(prompt, list):
+        return "\n".join([f"- {s.strip()}" for s in prompt] + ["\n"])
+
+    out = []
+    header_prefix = "#" * _header_depth
+    for k, v in prompt.items():
+        out.append(f"{header_prefix} {k}\n")
+        out.append(compile_prompt_to_md(v, _header_depth=_header_depth + 1))
+    return "\n".join(out)
+
+
+@dataclass
+class FunctionSpec(DataClassJsonMixin):
+    name: str
+    json_schema: dict  # JSON schema
+    description: str
+
+    def __post_init__(self):
+        # validate the schema
+        jsonschema.Draft7Validator.check_schema(self.json_schema)
+
+    @property
+    def as_openai_tool_dict(self):
+        """Convert to OpenAI's function format."""
+        return {
+            "type": "function",
+            "function": {
+                "name": self.name,
+                "description": self.description,
+                "parameters": self.json_schema,
+            },
+        }
+
+    @property
+    def openai_tool_choice_dict(self):
+        return {
+            "type": "function",
+            "function": {"name": self.name},
+        }
+
+    @property
+    def as_anthropic_tool_dict(self):
+        """Convert to Anthropic's tool format."""
+        return {
+            "name": self.name,
+            "description": self.description,
+            "input_schema": self.json_schema,  # Anthropic uses input_schema instead of parameters
+        }
+
+    @property
+    def anthropic_tool_choice_dict(self):
+        """Convert to Anthropic's tool choice format."""
+        return {
+            "type": "tool",  # Anthropic uses "tool" instead of "function"
+            "name": self.name,
+        }

aide/example_tasks/bitcoin_price.md

@@ -0,0 +1,5 @@
+## Goal
+Build a timeseries forcasting model for bitcoin close price.
+
+## Evaluation metric
+Root-Mean-Squared-Error (RMSE) between the logarithm of the predicted value and the logarithm of the observed price.

aide/example_tasks/bitcoin_price/BTC-USD.csv

@@ -0,0 +1,501 @@
+Date,Open,High,Low,Close,Adj Close,Volume
+2014-09-15,465.864014,468.174011,384.532013,398.821014,398.821014,156903400
+2014-09-22,399.100006,441.557007,374.332001,377.181000,377.181000,186772600
+2014-09-29,376.928009,391.378998,289.295990,320.510010,320.510010,276657896
+2014-10-06,320.389008,382.726013,302.559998,378.549011,378.549011,341152804
+2014-10-13,377.920990,411.697998,368.897003,389.545990,389.545990,156902070
+2014-10-20,389.230988,392.645996,342.877014,354.704010,354.704010,113691800
+2014-10-27,354.777008,359.984009,320.626007,325.891998,325.891998,107075700
+2014-11-03,325.569000,363.626007,325.076996,363.264008,363.264008,116793470
+2014-11-10,362.265015,457.092987,357.561005,387.881989,387.881989,208239500
+2014-11-17,388.348999,410.199005,344.112000,367.572998,367.572998,178181000
+2014-11-24,366.947998,394.700989,358.454987,378.046997,378.046997,131238570
+2014-12-01,378.248993,384.037994,365.756012,375.095001,375.095001,80679570
+2014-12-08,374.964996,376.028992,338.763000,351.631989,351.631989,141754100
+2014-12-15,351.360992,351.815002,304.231995,320.842987,320.842987,184757400
+2014-12-22,321.067993,336.286987,311.078003,317.239014,317.239014,107137540
+2014-12-29,317.700989,320.434998,257.612000,264.195007,264.195007,143354400
+2015-01-05,265.084015,298.753998,265.039001,265.660004,265.660004,164241500
+2015-01-12,266.145996,272.203003,171.509995,210.339005,210.339005,363112212
+2015-01-19,211.470993,255.074005,205.153000,253.718002,253.718002,189165500
+2015-01-26,254.078995,309.384003,212.014999,226.972000,226.972000,306840500
+2015-02-02,226.490997,245.957001,214.725006,223.412003,223.412003,183691200
+2015-02-09,223.389008,265.610992,215.332001,234.824997,234.824997,230343800
+2015-02-16,234.824997,255.320007,229.022003,235.977005,235.977005,154676000
+2015-02-23,235.994995,261.660004,232.421005,260.201996,260.201996,138892900
+2015-03-02,260.356995,285.795990,258.312988,274.354004,274.354004,242425100
+2015-03-09,274.812012,300.044006,273.893005,286.393005,286.393005,259502000
+2015-03-16,285.684998,294.112000,248.636002,267.959991,267.959991,206778200
+2015-03-23,267.894989,277.296997,236.514999,242.712997,242.712997,179497700
+2015-03-30,242.878998,260.674988,239.214005,260.597992,260.597992,150120700
+2015-04-06,260.721008,261.798004,232.770996,236.153000,236.153000,147866600
+2015-04-13,235.949997,236.934998,216.322998,222.600006,222.600006,158658500
+2015-04-20,222.612000,237.908997,214.873993,219.429993,219.429993,148576200
+2015-04-27,219.429001,243.240005,218.022995,240.358002,240.358002,162643200
+2015-05-04,240.356003,247.804001,228.572998,240.296005,240.296005,166059200
+2015-05-11,240.298996,244.270004,234.057007,236.802002,236.802002,130322100
+2015-05-18,236.886993,241.977997,231.817001,240.953003,240.953003,114746500
+2015-05-25,240.927002,241.020996,229.542007,230.190002,230.190002,107149900
+2015-06-01,230.233002,231.712997,221.296005,222.880997,222.880997,121536400
+2015-06-08,222.878998,234.858002,222.839005,233.542999,233.542999,121541200
+2015-06-15,233.421997,256.852997,233.421997,243.944000,243.944000,191537000
+2015-06-22,243.968994,251.339005,240.365005,249.011002,249.011002,115888900
+2015-06-29,248.720993,274.506012,248.580994,271.912994,271.912994,206668700
+2015-07-06,271.108002,314.394012,264.385986,310.867004,310.867004,353198700
+2015-07-13,310.826996,310.947998,272.042999,273.614014,273.614014,235861300
+2015-07-20,273.498993,293.052002,272.959991,292.686005,292.686005,157457100
+2015-07-27,292.639008,297.773987,277.528992,282.614014,282.614014,162700300
+2015-08-03,282.806000,285.714996,260.467987,265.083008,265.083008,207110300
+2015-08-10,265.477997,270.673004,257.040985,258.506989,258.506989,177043500
+2015-08-17,258.489990,260.505005,211.078995,228.169006,228.169006,221695600
+2015-08-24,228.112000,235.218994,199.567001,228.761002,228.761002,241915000
+2015-08-31,229.113998,242.912003,225.914993,239.839996,239.839996,144635800
+2015-09-07,239.934006,245.781006,229.332001,230.511993,230.511993,148588700
+2015-09-14,230.608994,259.182007,227.401993,231.212006,231.212006,126654700
+2015-09-21,231.216995,237.427002,225.117004,232.757004,232.757004,137307600
+2015-09-28,232.835999,239.802002,232.466995,238.259003,238.259003,136796200
+2015-10-05,238.147003,247.242996,237.035004,247.050003,247.050003,142478600
+2015-10-12,246.875000,273.578003,243.757004,261.643005,261.643005,199808300
+2015-10-19,261.860992,294.058990,260.950989,283.679993,283.679993,220696500
+2015-10-26,283.627991,334.169006,280.510010,325.431000,325.431000,357648800
+2015-11-02,325.941986,495.562012,323.209015,373.368011,373.368011,954934692
+2015-11-09,374.324005,385.278015,300.997009,320.165985,320.165985,484398204
+2015-11-16,319.734985,338.350006,312.217010,324.536011,324.536011,292569400
+2015-11-23,324.350006,371.938995,316.769989,371.294006,371.294006,337018400
+2015-11-30,371.437012,402.808990,349.464996,388.782990,388.782990,416857500
+2015-12-07,389.977997,469.102997,385.411011,433.755005,433.755005,562276900
+2015-12-14,433.272003,465.580994,430.455994,442.684998,442.684998,553061704
+2015-12-21,442.838013,458.455994,405.760010,422.822998,422.822998,441634396
+2015-12-28,423.342987,436.246002,418.480988,430.010986,430.010986,300130400
+2016-01-04,430.061005,462.933990,426.341003,447.990997,447.990997,319871700
+2016-01-11,448.697998,450.661987,354.914001,382.299011,382.299011,692913108
+2016-01-18,381.733002,425.266998,375.282013,402.971008,402.971008,493900500
+2016-01-25,402.316986,402.316986,365.451996,368.766998,368.766998,378185696
+2016-02-01,369.350006,391.608002,367.957001,376.619995,376.619995,337404904
+2016-02-08,376.756989,407.230011,372.898010,407.230011,407.230011,465920396
+2016-02-15,407.567993,448.045990,397.748993,438.798004,438.798004,546633900
+2016-02-22,438.989014,439.858002,413.907013,433.503998,433.503998,465584792
+2016-02-29,433.437988,441.506989,394.035004,407.707001,407.707001,628483380
+2016-03-07,407.756989,423.925995,406.308990,414.065002,414.065002,548114688
+2016-03-14,414.200989,420.997009,406.136993,413.755005,413.755005,520066400
+2016-03-21,413.417999,428.796997,410.381012,426.765015,426.765015,426699404
+2016-03-28,426.548004,426.856995,412.496002,420.903992,420.903992,405154204
+2016-04-04,421.299011,425.360992,416.515015,421.563995,421.563995,415027900
+2016-04-11,421.872009,432.625000,420.532990,427.398987,427.398987,382137900
+2016-04-18,427.610992,460.145996,427.085999,458.554993,458.554993,427064400
+2016-04-25,459.121002,467.964996,436.649994,451.875000,451.875000,492934504
+2016-05-02,451.933014,461.375000,441.776001,458.548004,458.548004,403817800
+2016-05-09,458.205994,462.480988,448.954010,457.567993,457.567993,351471700
+2016-05-16,457.585999,458.200012,437.389008,439.322998,439.322998,470557100
+2016-05-23,439.347992,553.960022,438.822998,526.232971,526.232971,741517392
+2016-05-30,528.471008,590.132019,520.661987,574.976990,574.976990,658668096
+2016-06-06,574.601990,684.843994,567.513977,672.783997,672.783997,757909588
+2016-06-13,671.653992,777.989990,662.804016,763.781006,763.781006,1553070776
+2016-06-20,763.927002,764.083984,558.138977,629.367004,629.367004,1464507992
+2016-06-27,629.348999,704.968018,620.523987,658.664001,658.664001,918252000
+2016-07-04,658.804016,683.661987,611.833984,649.359985,649.359985,1040574400
+2016-07-11,648.484009,682.364990,644.979980,679.458984,679.458984,682454600
+2016-07-18,679.809021,681.554993,646.721985,661.284973,661.284973,607681700
+2016-07-25,661.263000,661.828003,624.364990,624.681030,624.681030,747178496
+2016-08-01,624.601990,626.119019,531.333984,592.690002,592.690002,1015417588
+2016-08-08,592.736023,599.984009,564.781006,570.473022,570.473022,504473696
+2016-08-15,570.494019,584.158020,563.239990,581.307983,581.307983,364239500
+2016-08-22,581.310974,589.473999,568.630005,573.911987,573.911987,545508100
+2016-08-29,574.070984,611.836975,570.810974,608.633972,608.633972,670372392
+2016-09-05,608.989990,628.817993,600.505981,606.718994,606.718994,505899104
+2016-09-12,607.005005,611.952026,605.192017,609.874023,609.874023,417858304
+2016-09-19,609.870972,610.932007,595.786011,600.825989,600.825989,397773096
+2016-09-26,600.807007,615.237000,600.348999,610.892029,610.892029,364686600
+2016-10-03,610.968018,619.848999,609.479004,616.752014,616.752014,367150204
+2016-10-10,616.822021,642.898010,616.197021,641.630981,641.630981,462540104
+2016-10-17,641.817993,661.129028,628.013000,657.070984,657.070984,438931392
+2016-10-24,657.161011,720.401978,652.594971,701.864014,701.864014,654150896
+2016-10-31,702.640015,745.773010,678.156006,711.521973,711.521973,698826796
+2016-11-07,710.736023,740.046021,687.315002,702.031006,702.031006,554004996
+2016-11-14,701.997009,756.237000,699.807983,731.026001,731.026001,736991600
+2016-11-21,731.265015,753.869995,729.625000,732.034973,732.034973,528543596
+2016-11-28,732.484009,781.296021,732.484009,773.872009,773.872009,552642604
+2016-12-05,773.393982,777.091980,751.713013,769.731018,769.731018,578980796
+2016-12-12,770.039978,794.737000,770.039978,790.530029,790.530029,538898096
+2016-12-19,790.692017,925.117004,790.320007,896.182983,896.182983,1087074704
+2016-12-26,896.905029,1003.080017,896.898010,998.325012,998.325012,1161413112
+2017-01-02,998.617004,1191.099976,823.556030,911.198975,911.198975,2052639008
+2017-01-09,913.244019,919.447998,755.755981,821.797974,821.797974,1123984880
+2017-01-16,821.783020,937.525024,820.270996,924.672974,924.672974,883611600
+2017-01-23,925.499023,928.265991,891.687012,919.495972,919.495972,693151900
+2017-01-30,920.151001,1045.900024,919.473999,1027.339966,1027.339966,1009290288
+2017-02-06,1028.400024,1088.989990,946.690979,999.181030,999.181030,1227087016
+2017-02-13,998.885010,1061.099976,976.002014,1047.869995,1047.869995,763560200
+2017-02-20,1048.689941,1200.390015,1041.689941,1165.199951,1165.199951,1209106992
+2017-02-27,1163.780029,1280.310059,1163.380005,1267.119995,1267.119995,1546994008
+2017-03-06,1267.469971,1276.000000,1077.250000,1221.380005,1221.380005,2064090976
+2017-03-13,1221.780029,1257.979980,957.655029,1036.739990,1036.739990,3296504976
+2017-03-20,1037.239990,1122.430054,903.713013,966.724976,966.724976,2483811008
+2017-03-27,972.054993,1107.589966,971.984009,1102.170044,1102.170044,2601401984
+2017-04-03,1102.949951,1197.209961,1102.949951,1187.869995,1187.869995,2711437088
+2017-04-10,1187.300049,1208.069946,1156.439941,1182.939941,1182.939941,1714353040
+2017-04-17,1183.250000,1240.790039,1172.650024,1207.209961,1207.209961,1907337008
+2017-04-24,1209.630005,1347.910034,1209.630005,1347.890015,1347.890015,2620499984
+2017-05-01,1348.300049,1618.030029,1348.300049,1596.709961,1596.709961,5316902848
+2017-05-08,1596.920044,1873.930054,1596.920044,1808.910034,1808.910034,5981268000
+2017-05-15,1808.439941,2119.080078,1661.910034,2041.199951,2041.199951,6916110976
+2017-05-22,2043.189941,2763.709961,1855.829956,2155.800049,2155.800049,12064150016
+2017-05-29,2159.429932,2585.889893,2107.169922,2511.810059,2511.810059,9823705088
+2017-06-05,2512.399902,2999.909912,2510.219971,2958.110107,2958.110107,11378039936
+2017-06-12,2953.219971,2997.260010,2212.959961,2548.290039,2548.290039,11981910016
+2017-06-19,2549.030029,2772.010010,2549.030029,2589.409912,2589.409912,9130718976
+2017-06-26,2590.570068,2615.250000,2332.989990,2506.469971,2506.469971,7730852928
+2017-07-03,2498.560059,2916.139893,2480.469971,2518.439941,2518.439941,5831748992
+2017-07-10,2525.250000,2537.159912,1843.030029,1929.819946,1929.819946,7453121024
+2017-07-17,1932.619995,2900.699951,1932.619995,2730.399902,2730.399902,9947990080
+2017-07-24,2732.699951,2897.449951,2450.800049,2757.179932,2757.179932,6942860928
+2017-07-31,2763.239990,3293.290039,2668.590088,3213.939941,3213.939941,8137841920
+2017-08-07,3212.780029,4208.390137,3180.889893,4073.260010,4073.260010,13618979840
+2017-08-14,4066.100098,4484.700195,3906.179932,4087.659912,4087.659912,18573839872
+2017-08-21,4090.479980,4455.700195,3674.580078,4382.879883,4382.879883,15749739904
+2017-08-28,4384.450195,4975.040039,4224.640137,4582.959961,4582.959961,15582599936
+2017-09-04,4591.629883,4661.000000,3951.040039,4122.939941,4122.939941,15468230272
+2017-09-11,4122.470215,4344.649902,2946.620117,3582.879883,3582.879883,15563199872
+2017-09-18,3591.090088,4094.070068,3553.530029,3682.840088,3682.840088,9023459008
+2017-09-25,3681.580078,4403.740234,3681.580078,4403.740234,4403.740234,9599860096
+2017-10-02,4395.810059,4624.140137,4164.049805,4610.479980,4610.479980,8289027968
+2017-10-09,4614.520020,5852.479980,4564.250000,5678.189941,5678.189941,14840320000
+2017-10-16,5687.569824,6194.879883,5151.439941,6008.419922,6008.419922,14605609856
+2017-10-23,6006.000000,6255.709961,5397.879883,6153.850098,6153.850098,14982660096
+2017-10-30,6114.850098,7617.479980,6040.850098,7407.410156,7407.410156,19841690624
+2017-11-06,7403.220215,7776.419922,5519.009766,5950.069824,5950.069824,32340969984
+2017-11-13,5938.250000,8101.910156,5844.290039,8036.490234,8036.490234,30253229824
+2017-11-20,8039.069824,9522.929688,7762.709961,9330.549805,9330.549805,30501020160
+2017-11-27,9352.719727,11858.700195,9202.049805,11323.200195,11323.200195,50411559424
+2017-12-04,11315.400391,18353.400391,11081.799805,15455.400391,15455.400391,92115268608
+2017-12-11,15427.400391,20089.000000,15404.799805,19140.800781,19140.800781,93877198848
+2017-12-18,19106.400391,19371.000000,11833.000000,13925.799805,13925.799805,117256598528
+2017-12-25,13995.900391,16930.900391,12350.099609,14156.400391,14156.400391,88557499392
+2018-01-01,14112.200195,17712.400391,13154.700195,16477.599609,16477.599609,123814400000
+2018-01-08,16476.199219,16537.900391,13105.900391,13772.000000,13772.000000,106022199296
+2018-01-15,13767.299805,14445.500000,9402.290039,11600.099609,11600.099609,97932879872
+2018-01-22,11633.099609,12040.299805,10129.700195,11786.299805,11786.299805,64691999232
+2018-01-29,11755.500000,11875.599609,7796.490234,8277.009766,8277.009766,60810019840
+2018-02-05,8270.540039,9122.549805,6048.259766,8129.970215,8129.970215,62489091072
+2018-02-12,8141.430176,11349.799805,8141.430176,10551.799805,10551.799805,53626569728
+2018-02-19,10552.599609,11958.500000,9407.059570,9664.730469,9664.730469,55388419584
+2018-02-26,9669.429688,11528.200195,9501.730469,11512.599609,11512.599609,48902649856
+2018-03-05,11532.400391,11704.099609,8513.030273,9578.629883,9578.629883,49671590400
+2018-03-12,9602.929688,9937.500000,7397.990234,8223.679688,8223.679688,42075919360
+2018-03-19,8344.120117,9177.370117,8182.399902,8495.780273,8495.780273,40853020160
+2018-03-26,8498.469727,8530.080078,6526.870117,6844.229980,6844.229980,37970689536
+2018-04-02,6844.859863,7530.939941,6575.000000,7023.520020,7023.520020,31804380416
+2018-04-09,7044.319824,8338.419922,6661.990234,8329.110352,8329.110352,40915320320
+2018-04-16,8337.570313,9001.639648,7881.720215,8802.459961,8802.459961,48741869568
+2018-04-23,8794.389648,9745.320313,8727.089844,9419.080078,9419.080078,61882420224
+2018-04-30,9426.110352,9964.500000,8891.049805,9654.799805,9654.799805,57244449280
+2018-05-07,9645.669922,9665.849609,8223.500000,8723.940430,8723.940430,50119759872
+2018-05-14,8713.099609,8881.120117,7974.819824,8513.250000,8513.250000,42360260096
+2018-05-21,8522.330078,8557.519531,7270.959961,7368.220215,7368.220215,35808230144
+2018-05-28,7371.310059,7754.890137,7090.680176,7720.250000,7720.250000,35465450496
+2018-06-04,7722.529785,7753.819824,6709.069824,6786.020020,6786.020020,33010609408
+2018-06-11,6799.290039,6910.180176,6285.629883,6499.270020,6499.270020,29844289792
+2018-06-18,6510.069824,6822.500000,5826.410156,6173.229980,6173.229980,28592079872
+2018-06-25,6171.970215,6465.509766,5835.750000,6385.819824,6385.819824,28842940160
+2018-07-02,6380.379883,6885.910156,6305.700195,6773.879883,6773.879883,29906419968
+2018-07-09,6775.080078,6838.680176,6136.419922,6359.640137,6359.640137,25200120064
+2018-07-16,6357.009766,7594.669922,6357.009766,7418.490234,7418.490234,34261730048
+2018-07-23,7414.709961,8424.269531,7409.100098,8218.459961,8218.459961,36446479872
+2018-07-30,8221.580078,8235.500000,6940.700195,7068.479980,7068.479980,32425310000
+2018-08-06,7062.939941,7166.549805,6109.029785,6322.689941,6322.689941,32181950000
+2018-08-13,6341.359863,6617.350098,5971.049805,6506.069824,6506.069824,30898230000
+2018-08-20,6500.509766,6816.790039,6297.930176,6707.259766,6707.259766,25842490000
+2018-08-27,6710.799805,7306.310059,6689.709961,7272.720215,7272.720215,30229310000
+2018-09-03,7279.029785,7388.430176,6197.520020,6300.859863,6300.859863,31456960000
+2018-09-10,6301.569824,6596.100098,6260.209961,6517.180176,6517.180176,26405400000
+2018-09-17,6514.060059,6814.560059,6208.339844,6710.629883,6710.629883,32109420000
+2018-09-24,6704.770020,6785.029785,6381.859863,6625.560059,6625.560059,31328000000
+2018-10-01,6619.850098,6653.299805,6454.029785,6602.950195,6602.950195,25943820000
+2018-10-08,6600.189941,6675.060059,6236.470215,6290.930176,6290.930176,26462410000
+2018-10-15,6292.640137,6965.060059,6258.680176,6482.350098,6482.350098,29669680000
+2018-10-22,6486.049805,6543.799805,6447.029785,6486.390137,6486.390137,24188720000
+2018-10-29,6492.350098,6547.140137,6294.569824,6376.129883,6376.129883,28245180000
+2018-11-05,6363.620117,6552.160156,6350.169922,6411.270020,6411.270020,30472560000
+2018-11-12,6411.759766,6434.209961,5358.379883,5623.540039,5623.540039,36972800000
+2018-11-19,5620.779785,5620.779785,3585.060059,4009.969971,4009.969971,42533970000
+2018-11-26,4015.070068,4413.020508,3643.919922,4139.877930,4139.877930,42945276472
+2018-12-03,4147.323730,4155.979492,3280.228760,3614.234375,3614.234375,38386467708
+2018-12-10,3612.046387,3647.332520,3191.303467,3252.839111,3252.839111,29869247431
+2018-12-17,3253.123047,4198.429688,3253.123047,3998.980225,3998.980225,46021506218
+2018-12-24,4000.331787,4271.792480,3642.632080,3865.952637,3865.952637,39249734918
+2018-12-31,3866.839111,4093.297363,3707.231201,4076.632568,4076.632568,34343716582
+2019-01-07,4078.584961,4109.020996,3544.927246,3552.953125,3552.953125,37523454177
+2019-01-14,3557.311035,3758.533447,3552.285156,3601.013672,3601.013672,38588596987
+2019-01-21,3600.372803,3654.933105,3539.721436,3583.965820,3583.965820,36949379050
+2019-01-28,3584.283203,3586.750977,3400.819824,3464.013428,3464.013428,40130888766
+2019-02-04,3467.211670,3695.614014,3391.023682,3690.188232,3690.188232,41224140801
+2019-02-11,3695.613037,3695.613037,3607.697754,3673.836182,3673.836182,44533597993
+2019-02-18,3671.369873,4210.641602,3669.982422,3810.427490,3810.427490,63853356366
+2019-02-25,3807.002441,3913.707275,3787.058838,3847.175781,3847.175781,56444684748
+2019-03-04,3845.091553,3987.237793,3733.749756,3951.599854,3951.599854,69110769320
+2019-03-11,3953.740234,4077.036377,3863.559082,4025.229004,4025.229004,67357766371
+2019-03-18,4029.968506,4097.359863,4005.151367,4022.168213,4022.168213,67975640775
+2019-03-25,4024.112793,4296.806641,3934.031250,4105.404297,4105.404297,71015021591
+2019-04-01,4105.362305,5307.003418,4096.901367,5198.896973,5198.896973,123047080530
+2019-04-08,5199.835449,5421.651367,4955.852539,5167.722168,5167.722168,98826873355
+2019-04-15,5167.321777,5359.924805,5024.069336,5314.531250,5314.531250,90285516253
+2019-04-22,5312.494629,5642.044434,5177.368652,5285.139160,5285.139160,105590631334
+2019-04-29,5284.858398,5886.893555,5216.487793,5795.708496,5795.708496,107035835884
+2019-05-06,5791.693359,7503.872070,5653.687500,6972.371582,6972.371582,141929602925
+2019-05-13,6971.178223,8320.824219,6898.282227,8197.689453,8197.689453,199543788416
+2019-05-20,8196.923828,8687.520508,7533.196777,8673.215820,8673.215820,173001140580
+2019-05-27,8674.072266,9008.314453,8172.550781,8742.958008,8742.958008,173016477871
+2019-06-03,8741.747070,8743.500000,7564.488770,7688.077148,7688.077148,140124650113
+2019-06-10,7692.284668,9335.867188,7586.730957,8994.488281,8994.488281,135051141522
+2019-06-17,8988.923828,11246.144531,8988.923828,10855.371094,10855.371094,136422336191
+2019-06-24,10853.744141,13796.489258,10491.852539,10817.155273,10817.155273,221502737682
+2019-07-01,10796.930664,12006.075195,9737.884766,11450.846680,11450.846680,181410821667
+2019-07-08,11446.596680,13129.529297,10234.576172,10256.058594,10256.058594,180936684097
+2019-07-15,10257.838867,11052.766602,9163.134766,10599.105469,10599.105469,157356814406
+2019-07-22,10596.948242,10651.791016,9252.296875,9552.860352,9552.860352,112459229432
+2019-07-29,9548.178711,11009.207031,9437.335938,10970.184570,10970.184570,110790788891
+2019-08-05,10960.735352,12273.821289,10960.735352,11523.579102,11523.579102,141427393788
+2019-08-12,11528.189453,11528.189453,9675.316406,10345.810547,10345.810547,120224711798
+2019-08-19,10350.283203,10947.041992,9831.462891,10138.517578,10138.517578,112893851548
+2019-08-26,10126.299805,10512.328125,9421.629883,9757.970703,9757.970703,104346358671
+2019-09-02,9757.473633,10898.761719,9730.650391,10441.276367,10441.276367,116441433312
+2019-09-09,10443.228516,10450.311523,9980.776367,10347.712891,10347.712891,102876391724
+2019-09-16,10347.222656,10386.867188,9851.692383,10070.392578,10070.392578,107930839412
+2019-09-23,10067.962891,10074.238281,7895.629395,8104.185547,8104.185547,124735468740
+2019-09-30,8104.226563,8497.692383,7830.758789,7988.155762,7988.155762,97716137179
+2019-10-07,7989.120605,8721.780273,7905.766113,8321.005859,8321.005859,118550918408
+2019-10-14,8320.832031,8410.714844,7902.164063,8222.078125,8222.078125,105710167423
+2019-10-21,8225.115234,10021.744141,7446.988770,9551.714844,9551.714844,176678165128
+2019-10-28,9565.101563,9805.118164,9028.717773,9235.354492,9235.354492,180364809056
+2019-11-04,9235.607422,9505.051758,8775.534180,9055.526367,9055.526367,160702732609
+2019-11-11,9056.917969,9081.279297,8473.973633,8577.975586,8577.975586,134166660428
+2019-11-18,8573.980469,8653.280273,6936.706543,7047.916992,7047.916992,171626385789
+2019-11-25,7039.977051,7836.102051,6617.166992,7424.292480,7424.292480,162444865040
+2019-12-02,7424.036133,7743.431641,7170.922852,7564.345215,7564.345215,121327747581
+2019-12-09,7561.795410,7618.091797,6924.375977,7152.301758,7152.301758,122542519854
+2019-12-16,7153.663086,7518.033203,6540.049316,7511.588867,7511.588867,165399102665
+2019-12-23,7508.902344,7656.176270,7189.934082,7422.652832,7422.652832,161758217053
+2019-12-30,7420.272949,7544.497070,6914.996094,7411.317383,7411.317383,149690652648
+2020-01-06,7410.452148,8396.738281,7409.292969,8192.494141,8192.494141,184901289966
+2020-01-13,8189.771973,9164.362305,8079.700684,8706.245117,8706.245117,241668743794
+2020-01-20,8704.631836,8792.994141,8266.840820,8596.830078,8596.830078,165113602391
+2020-01-27,8597.308594,9553.125977,8597.308594,9344.365234,9344.365234,212298356923
+2020-02-03,9344.683594,10129.435547,9112.811523,10116.673828,10116.673828,239180811399
+2020-02-10,10115.559570,10457.626953,9722.386719,9934.433594,9934.433594,300413081965
+2020-02-17,9936.560547,10191.675781,9507.637695,9924.515625,9924.515625,303140360741
+2020-02-24,9921.583008,9951.746094,8471.212891,8562.454102,8562.454102,299233009521
+2020-03-02,8563.264648,9167.695313,8105.252930,8108.116211,8108.116211,276706069337
+2020-03-09,8111.146484,8177.793457,4106.980957,5392.314941,5392.314941,326123224372
+2020-03-16,5385.229492,6844.261719,4575.357910,5830.254883,5830.254883,309907233901
+2020-03-23,5831.374512,6892.511230,5785.004395,5922.042969,5922.042969,272468247401
+2020-03-30,5925.538574,7088.247559,5903.234375,6791.129395,6791.129395,259568096070
+2020-04-06,6788.049805,7427.939453,6782.889648,6971.091797,6971.091797,274123270580
+2020-04-13,6965.616699,7280.521973,6555.504395,7189.424805,7189.424805,248073118275
+2020-04-20,7186.873535,7700.594238,6834.442383,7679.867188,7679.867188,247735013853
+2020-04-27,7679.418945,9440.650391,7679.418945,8897.468750,8897.468750,327820350244
+2020-05-04,8895.745117,9996.743164,8395.107422,8756.430664,8756.430664,361023995652
+2020-05-11,8755.535156,9823.001953,8374.323242,9670.739258,9670.739258,325655447393
+2020-05-18,9675.695313,9906.030273,8787.250977,8790.368164,8790.368164,247011272897
+2020-05-25,8786.107422,9704.030273,8719.667969,9461.058594,9461.058594,221372861711
+2020-06-01,9463.605469,10199.565430,9450.899414,9758.852539,9758.852539,194228716225
+2020-06-08,9760.063477,9938.297852,9263.069336,9386.788086,9386.788086,160324519712
+2020-06-15,9386.035156,9579.430664,8990.175781,9303.629883,9303.629883,138300100692
+2020-06-22,9300.915039,9680.367188,8975.525391,9143.582031,9143.582031,125863637229
+2020-06-29,9140.029297,9309.754883,8977.015625,9073.942383,9073.942383,102779717630
+2020-07-06,9072.849609,9450.335938,9058.664063,9276.500000,9276.500000,113994286210
+2020-07-13,9277.205078,9306.405273,9088.947266,9185.817383,9185.817383,106299733734
+2020-07-20,9187.220703,10023.807617,9137.509766,9905.166992,9905.166992,120174678243
+2020-07-27,9905.217773,12034.144531,9903.969727,11053.614258,11053.614258,188247005753
+2020-08-03,11043.768555,11902.335938,11012.415039,11675.739258,11675.739258,147527884833
+2020-08-10,11662.256836,12150.994141,11195.708984,11892.803711,11892.803711,173916901671
+2020-08-17,11895.658203,12359.056641,11448.805664,11664.847656,11664.847656,161417688724
+2020-08-24,11663.689453,11807.631836,11185.941406,11711.505859,11711.505859,149742951013
+2020-08-31,11713.306641,12067.081055,9946.675781,10280.351563,10280.351563,221515306287
+2020-09-07,10280.998047,10578.837891,9916.493164,10323.755859,10323.755859,264126813725
+2020-09-14,10328.734375,11134.092773,10266.008789,10938.271484,10938.271484,210690462551
+2020-09-21,10934.925781,10988.304688,10197.865234,10775.269531,10775.269531,263386919085
+2020-09-28,10776.613281,10945.347656,10416.689453,10669.583008,10669.583008,343112501609
+2020-10-05,10676.529297,11442.210938,10528.890625,11384.181641,11384.181641,256921249410
+2020-10-12,11392.635742,11698.467773,11223.012695,11483.359375,11483.359375,162045277956
+2020-10-19,11495.038086,13329.183594,11408.290039,13031.173828,13031.173828,210845277262
+2020-10-26,13031.201172,14028.213867,12822.382813,13737.109375,13737.109375,240919962782
+2020-11-02,13737.032227,15903.437500,13243.160156,15479.567383,15479.567383,238108964192
+2020-11-09,15479.595703,16463.177734,14865.529297,15955.587891,15955.587891,206407256877
+2020-11-16,15955.577148,18936.621094,15880.706055,18370.001953,18370.001953,274506990351
+2020-11-23,18370.017578,19390.964844,16351.035156,18177.484375,18177.484375,301939363508
+2020-11-30,18178.322266,19845.974609,18178.322266,19345.121094,19345.121094,253088772491
+2020-12-07,19343.128906,19411.828125,17619.533203,19142.382813,19142.382813,193678842258
+2020-12-14,19144.492188,24209.660156,19012.708984,23477.294922,23477.294922,281723269352
+2020-12-21,23474.455078,28288.839844,22159.367188,26272.294922,26272.294922,339132207025
+2020-12-28,26280.822266,34608.558594,25987.298828,32782.023438,32782.023438,379626054545
+2021-01-04,32810.949219,41946.738281,28722.755859,38356.441406,38356.441406,538834804975
+2021-01-11,38346.531250,39966.406250,30549.599609,35791.277344,35791.277344,508900951412
+2021-01-18,35792.238281,37755.890625,28953.373047,32289.378906,32289.378906,423439379849
+2021-01-25,32285.798828,38406.261719,29367.138672,33114.359375,33114.359375,495037340009
+2021-02-01,33114.578125,40846.546875,32384.228516,38903.441406,38903.441406,449918619842
+2021-02-08,38886.828125,49487.640625,38076.324219,48717.289063,48717.289063,580021243858
+2021-02-15,48696.535156,58330.570313,46347.476563,57539.945313,57539.945313,470533297170
+2021-02-22,57532.738281,57533.390625,43241.617188,45137.769531,45137.769531,766679775805
+2021-03-01,45159.503906,52535.136719,45115.093750,51206.691406,51206.691406,333113779409
+2021-03-08,51174.117188,61683.863281,49506.054688,59302.316406,59302.316406,373838576722
+2021-03-15,59267.429688,60540.992188,53555.027344,57523.421875,57523.421875,393542542227
+2021-03-22,57517.890625,58471.480469,50856.570313,55950.746094,55950.746094,403128836645
+2021-03-29,55947.898438,60267.187500,55139.339844,58758.554688,58758.554688,408348561550
+2021-04-05,58760.875000,61276.664063,55604.023438,60204.964844,60204.964844,406637390979
+2021-04-12,60175.945313,64863.097656,52829.535156,56216.183594,56216.183594,508118943571
+2021-04-19,56191.585938,57520.054688,47159.484375,49004.253906,49004.253906,444719708699
+2021-04-26,49077.792969,58448.339844,48852.796875,56631.078125,56631.078125,335231529997
+2021-05-03,56620.273438,59464.613281,52969.054688,58232.316406,58232.316406,458766142208
+2021-05-10,58250.871094,59519.355469,43963.351563,46456.058594,46456.058594,483967915938
+2021-05-17,46415.898438,46623.558594,30681.496094,34770.582031,34770.582031,563629210102
+2021-05-24,34700.363281,40782.078125,33520.738281,35678.128906,35678.128906,350206489990
+2021-05-31,35658.593750,39478.953125,34241.945313,35862.378906,35862.378906,248884892724
+2021-06-07,35835.265625,39322.781250,31114.443359,39097.859375,39097.859375,298428017340
+2021-06-14,39016.968750,41295.269531,33432.074219,35698.296875,35698.296875,269949570053
+2021-06-21,35641.144531,35721.640625,28893.621094,34649.644531,34649.644531,305541799334
+2021-06-28,34679.121094,36542.109375,32770.679688,35287.781250,35287.781250,231729218470
+2021-07-05,35284.343750,35284.343750,32133.183594,34240.187500,34240.187500,178445862441
+2021-07-12,34254.015625,34592.156250,31100.673828,31796.810547,31796.810547,147501894180
+2021-07-19,31800.011719,35364.925781,29360.955078,35350.187500,35350.187500,156414750211
+2021-07-26,35384.031250,42541.679688,35287.312500,39974.894531,39974.894531,237553113912
+2021-08-02,39907.261719,45282.351563,37458.003906,43798.117188,43798.117188,226902699564
+2021-08-09,43791.925781,48098.683594,42848.687500,47047.003906,47047.003906,234268000892
+2021-08-16,47019.960938,49717.019531,43998.316406,49321.652344,49321.652344,236289722726
+2021-08-23,49291.675781,50482.078125,46394.281250,48829.832031,48829.832031,223947952688
+2021-08-30,48834.851563,51868.679688,46562.437500,51753.410156,51753.410156,256225023619
+2021-09-06,51769.003906,52853.765625,43285.207031,46063.269531,46063.269531,293311066613
+2021-09-13,46057.214844,48791.781250,43591.320313,47260.218750,47260.218750,226141953163
+2021-09-20,47261.406250,47328.199219,39787.609375,43208.539063,43208.539063,270099994433
+2021-09-27,43234.183594,49130.691406,40829.667969,48199.953125,48199.953125,223042115392
+2021-10-04,48208.906250,56401.304688,47045.003906,54771.578125,54771.578125,261919545521
+2021-10-11,54734.125000,62757.128906,54370.972656,61553.617188,61553.617188,277084548345
+2021-10-18,61548.804688,66930.390625,59643.343750,60930.835938,60930.835938,257856648064
+2021-10-25,60893.925781,63729.324219,58206.917969,61318.957031,61318.957031,256114057651
+2021-11-01,61320.449219,64242.792969,59695.183594,63326.988281,63326.988281,227064607869
+2021-11-08,63344.066406,68789.625000,62333.914063,65466.839844,65466.839844,259776276520
+2021-11-15,65521.289063,66281.570313,55705.179688,58730.476563,58730.476563,253419951009
+2021-11-22,58706.847656,59367.968750,53569.765625,57248.457031,57248.457031,243930001011
+2021-11-29,57291.906250,59113.402344,42874.617188,49368.847656,49368.847656,276690611812
+2021-12-06,49413.480469,51934.781250,46942.347656,50098.335938,50098.335938,208148497532
+2021-12-13,50114.742188,50205.000000,45598.441406,46707.015625,46707.015625,214770398202
+2021-12-20,46707.062500,51814.027344,45579.808594,50809.515625,50809.515625,175052499792
+2021-12-27,50802.609375,51956.328125,45819.953125,47345.218750,47345.218750,203998848909
+2022-01-03,47343.542969,47510.726563,40672.277344,41911.601563,41911.601563,276182787152
+2022-01-10,41910.230469,44278.421875,39796.570313,43113.878906,43113.878906,199473804544
+2022-01-17,43118.121094,43413.023438,34349.250000,36276.804688,36276.804688,196326044073
+2022-01-24,36275.734375,38825.410156,33184.058594,37917.601563,37917.601563,178727434896
+2022-01-31,37920.281250,42500.785156,36375.539063,42412.433594,42412.433594,143977109319
+2022-02-07,42406.781250,45661.171875,41748.156250,42197.515625,42197.515625,176958095731
+2022-02-14,42157.398438,44667.218750,38112.812500,38431.378906,38431.378906,144975794552
+2022-02-21,38423.210938,40005.347656,34459.218750,37709.785156,37709.785156,190469710084
+2022-02-28,37706.000000,45077.578125,37518.214844,38419.984375,38419.984375,187557375751
+2022-03-07,38429.304688,42465.671875,37260.203125,37849.664063,37849.664063,175966999156
+2022-03-14,37846.316406,42316.554688,37680.734375,41247.824219,41247.824219,184097042034
+2022-03-21,41246.132813,46827.546875,40668.042969,46820.492188,46820.492188,188591889758
+2022-03-28,46821.851563,48086.835938,44403.140625,46453.566406,46453.566406,223334181931
+2022-04-04,46445.273438,47106.140625,42021.207031,42207.671875,42207.671875,188557001876
+2022-04-11,42201.039063,42424.589844,39373.058594,39716.953125,39716.953125,174652159709
+2022-04-18,39721.203125,42893.582031,38696.191406,39469.292969,39469.292969,184314843516
+2022-04-25,39472.605469,40713.890625,37585.789063,38469.093750,38469.093750,216681007567
+2022-05-02,38472.187500,39902.949219,33878.964844,34059.265625,34059.265625,239044762282
+2022-05-09,34060.015625,34222.074219,26350.490234,31305.113281,31305.113281,357800928090
+2022-05-16,31304.375000,31305.341797,28708.955078,30323.722656,30323.722656,196429842104
+2022-05-23,30309.396484,30590.585938,28261.906250,29445.957031,29445.957031,212594819496
+2022-05-30,29443.365234,32249.863281,29303.572266,29906.662109,29906.662109,203063587161
+2022-06-06,29910.283203,31693.291016,26762.648438,26762.648438,26762.648438,215929645934
+2022-06-13,26737.578125,26795.589844,17708.623047,20553.271484,20553.271484,309685915250
+2022-06-20,20553.371094,21783.724609,19689.169922,21027.294922,21027.294922,175909056122
+2022-06-27,21028.238281,21478.089844,18729.656250,19297.076172,19297.076172,157426002965
+2022-07-04,19297.314453,22314.941406,19063.066406,20860.449219,20860.449219,206953871021
+2022-07-11,20856.353516,21600.640625,18999.953125,20779.343750,20779.343750,187297974219
+2022-07-18,20781.912109,24196.818359,20781.912109,22609.164063,22609.164063,244312089708
+2022-07-25,22607.156250,24572.580078,20776.816406,23336.896484,23336.896484,223759636595
+2022-08-01,23336.718750,23578.650391,22485.701172,23175.890625,23175.890625,166393135564
+2022-08-08,23179.527344,24974.914063,22771.519531,24319.333984,24319.333984,195343016900
+2022-08-15,24318.316406,25135.589844,20856.730469,21534.121094,21534.121094,208764012537
+2022-08-22,21531.462891,21804.908203,19616.814453,19616.814453,19616.814453,223346042309
+2022-08-29,19615.154297,20542.644531,19600.785156,19986.712891,19986.712891,208511390280
+2022-09-05,19988.789063,21770.552734,18644.466797,21769.255859,21769.255859,259528892947
+2022-09-12,21770.148438,22673.820313,19387.492188,19419.505859,19419.505859,261900187496
+2022-09-19,19418.572266,19674.630859,18290.314453,18802.097656,18802.097656,252873598365
+2022-09-26,18803.900391,20338.455078,18553.296875,19044.107422,19044.107422,280290121527
+2022-10-03,19044.068359,20408.392578,19025.226563,19446.425781,19446.425781,196809212801
+2022-10-10,19446.416016,19889.146484,18319.822266,19268.093750,19268.093750,197940078443
+2022-10-17,19268.562500,19666.994141,18770.970703,19567.007813,19567.007813,175664450104
+2022-10-24,19567.769531,20988.394531,19206.324219,20635.603516,20635.603516,302335723765
+2022-10-31,20633.695313,21446.886719,20086.240234,20926.486328,20926.486328,321270158405
+2022-11-07,20924.621094,21053.246094,15682.692383,16353.365234,16353.365234,471409252759
+2022-11-14,16352.028320,17109.324219,15872.941406,16291.832031,16291.832031,212305927001
+2022-11-21,16291.223633,16771.474609,15599.046875,16444.626953,16444.626953,184366390569
+2022-11-28,16440.222656,17197.498047,16054.530273,17130.486328,17130.486328,156325682750
+2022-12-05,17128.894531,17378.152344,16750.558594,17104.193359,17104.193359,129428712529
+2022-12-12,17102.500000,18318.531250,16584.701172,16757.976563,16757.976563,142170798266
+2022-12-19,16759.041016,17012.984375,16398.136719,16841.986328,16841.986328,107997970888
+2022-12-26,16842.250000,16959.845703,16408.474609,16625.080078,16625.080078,95526200508
+2023-01-02,16625.509766,17091.144531,16572.228516,17091.144531,17091.144531,90012614323
+2023-01-09,17093.992188,21075.142578,17093.992188,20880.798828,20880.798828,175267920183
+2023-01-16,20882.224609,23282.347656,20541.544922,22720.416016,22720.416016,188938770069
+2023-01-23,22721.087891,23919.890625,22406.076172,23774.566406,23774.566406,177486927537
+2023-01-30,23774.648438,24167.210938,22657.582031,22955.666016,22955.666016,171080243769
+2023-02-06,22954.021484,23367.958984,21539.392578,21788.203125,21788.203125,170212590188
+2023-02-13,21787.000000,25134.117188,21460.087891,24327.642578,24327.642578,209050300776
+2023-02-20,24336.623047,25126.851563,22861.558594,23561.212891,23561.212891,180472737469
+2023-02-27,23561.451172,23880.632813,22198.980469,22435.513672,22435.513672,138790785900
+2023-03-06,22436.816406,22584.292969,19628.253906,22163.949219,22163.949219,192057466346
+2023-03-13,22156.406250,28440.560547,21918.199219,28038.675781,28038.675781,305833103060
+2023-03-20,28041.601563,28803.335938,26759.996094,27994.330078,27994.330078,191720356212
+2023-03-27,27994.068359,29159.902344,26606.689453,28199.308594,28199.308594,121255230918
+2023-04-03,28183.080078,28739.238281,27276.720703,28333.050781,28333.050781,98141424556
+2023-04-10,28336.027344,31005.607422,28189.271484,30315.355469,30315.355469,122998807738
+2023-04-17,30317.146484,30470.302734,27169.570313,27591.384766,27591.384766,129935327803
+2023-04-24,27591.730469,29995.837891,27070.849609,29268.806641,29268.806641,137303647007
+2023-05-01,29227.103516,29820.126953,27680.792969,28454.978516,28454.978516,114911964253
+2023-05-08,28450.457031,28663.271484,25878.429688,26930.638672,26930.638672,109959496438
+2023-05-15,26931.384766,27646.347656,26415.101563,26753.826172,26753.826172,84459727715
+2023-05-22,26749.892578,28193.449219,25890.593750,28085.646484,28085.646484,90053064704
+2023-05-29,28075.591797,28432.039063,26574.644531,27119.066406,27119.066406,91351657572
+2023-06-05,27123.109375,27332.181641,25434.867188,25940.167969,25940.167969,116498927148
+2023-06-12,25934.285156,26769.394531,24797.167969,26336.212891,26336.212891,92905085602
+2023-06-19,26335.441406,31389.539063,26312.832031,30480.261719,30480.261719,138005623534
+2023-06-26,30480.523438,31256.863281,29600.275391,30620.769531,30620.769531,106681707513
+2023-07-03,30624.515625,31460.052734,29777.285156,30171.234375,30171.234375,90491031635
+2023-07-10,30172.423828,31814.515625,29966.386719,30249.132813,30249.132813,102917922458
+2023-07-17,30249.626953,30336.400391,29556.427734,30084.539063,30084.539063,81229098742
+2023-07-24,30081.662109,30093.394531,28934.294922,29275.308594,29275.308594,76309629498
+2023-07-31,29278.314453,29987.998047,28657.023438,29041.855469,29041.855469,87827001052
+2023-08-07,29038.513672,30176.796875,28724.140625,29282.914063,29282.914063,85153014454
+2023-08-14,29283.263672,29660.253906,25409.111328,26189.583984,26189.583984,116418274959
+2023-08-21,26188.691406,26786.898438,25520.728516,26089.693359,26089.693359,83086807452
+2023-08-28,26089.615234,28089.337891,25362.609375,25969.566406,25969.566406,113161627781
+2023-09-04,25968.169922,26414.005859,25404.359375,25832.226563,25832.226563,69814611152
+2023-09-11,25831.714844,26840.498047,24930.296875,26534.187500,26534.187500,85796699860
+2023-09-18,26532.994141,27488.763672,26221.050781,26256.826172,26256.826172,82251905213
+2023-09-25,26253.775391,28047.238281,26011.468750,27983.750000,27983.750000,73012240734
+2023-10-02,27976.798828,28494.458984,27215.552734,27935.089844,27935.089844,82183775698
+2023-10-09,27934.472656,27989.470703,26558.320313,27159.652344,27159.652344,72673654487
+2023-10-16,27162.628906,30287.482422,27130.472656,29993.896484,29993.896484,113402383094
+2023-10-23,30140.685547,35150.433594,30097.828125,34538.480469,34538.480469,165718773022
+2023-10-30,34531.742188,35919.843750,34083.308594,35049.355469,35049.355469,115520056635
+2023-11-06,35044.789063,37926.257813,34545.816406,37054.519531,37054.519531,134767494805
+2023-11-13,37070.304688,37964.894531,34948.500000,37386.546875,37386.546875,143535361089
+2023-11-20,37374.074219,38415.339844,35670.972656,37479.121094,37479.121094,130439893957
+2023-11-27,37454.191406,40135.605469,36750.128906,39978.390625,39978.390625,134360107955
+2023-12-04,39978.628906,44705.515625,39978.628906,43779.699219,43779.699219,188503614895
+2023-12-11,43792.019531,43808.375000,40234.578125,41364.664063,41364.664063,168493481950
+2023-12-18,41348.203125,44367.957031,40530.257813,43016.117188,43016.117188,152084171491
+2023-12-25,43010.574219,43804.781250,41424.062500,42265.187500,42265.187500,157807126208
+2024-01-01,42280.234375,45899.707031,40813.535156,43943.097656,43943.097656,202311773985
+2024-01-08,43948.707031,48969.371094,41724.613281,41796.269531,41796.269531,259971819745
+2024-01-15,41715.066406,43566.273438,40297.457031,41545.785156,41545.785156,139135824835
+2024-01-22,41553.652344,42797.175781,38521.894531,42035.593750,42035.593750,155314602893
+2024-01-29,42030.914063,43838.945313,41818.332031,42583.582031,42583.582031,135184187433
+2024-02-05,42577.621094,48535.937500,42264.816406,48293.917969,48293.917969,157826395448
+2024-02-12,48296.386719,52820.066406,47745.761719,52122.546875,52122.546875,213560041471
+2024-02-19,52134.812500,52945.050781,50561.777344,51733.238281,51733.238281,160769145627
+2024-02-26,51730.539063,63913.132813,50931.031250,63167.371094,63167.371094,322895666089
+2024-03-04,63137.003906,70083.054688,59323.910156,69019.789063,69019.789063,405709693297
+2024-03-11,69020.546875,73750.070313,64545.316406,68390.625000,68390.625000,405957750577
+2024-03-18,68371.304688,68897.132813,60807.785156,67234.171875,67234.171875,328097121682
+2024-03-25,67234.093750,71727.687500,66414.835938,71333.648438,71333.648438,216324625836
+2024-04-01,71333.484375,71342.093750,68110.695313,69702.148438,69702.148438,34873527352
+2024-04-02,69694.046875,69694.046875,64610.968750,66042.132813,66042.132813,49630113792

aide/example_tasks/house_prices.md

@@ -0,0 +1,24 @@
+## Goal
+It is your job to predict the sales price for each house. For each Id in the test set, you must predict the value of the SalePrice variable.
+
+## Background
+Ask a home buyer to describe their dream house, and they probably won't begin with the height of the basement ceiling or the proximity to an east-west railroad. But this playground competition's dataset proves that much more influences price negotiations than the number of bedrooms or a white-picket fence.
+With 79 explanatory variables describing (almost) every aspect of residential homes in Ames, Iowa, this competition challenges you to predict the final price of each home.
+
+## Evaluation
+Submissions are evaluated on [Root-Mean-Squared-Error (RMSE)](https://en.wikipedia.org/wiki/Root-mean-square_deviation) between the logarithm of the predicted value and the logarithm of the observed sales price. (Taking logs means that errors in predicting expensive houses and cheap houses will affect the result equally.)
+
+The file should contain a header and have the following format:
+
+```
+Id,SalePrice
+1461,169000.1
+1462,187724.1233
+1463,175221
+etc.
+```
+
+## Data description
+- **train.csv** - the training set
+- **test.csv** - the test set
+- **data_description.txt** - full description of each column, originally prepared by Dean De Cock but lightly edited to match the column names used here

aide/example_tasks/house_prices/data_description.txt

@@ -0,0 +1,523 @@
+MSSubClass: Identifies the type of dwelling involved in the sale.	
+
+        20	1-STORY 1946 & NEWER ALL STYLES
+        30	1-STORY 1945 & OLDER
+        40	1-STORY W/FINISHED ATTIC ALL AGES
+        45	1-1/2 STORY - UNFINISHED ALL AGES
+        50	1-1/2 STORY FINISHED ALL AGES
+        60	2-STORY 1946 & NEWER
+        70	2-STORY 1945 & OLDER
+        75	2-1/2 STORY ALL AGES
+        80	SPLIT OR MULTI-LEVEL
+        85	SPLIT FOYER
+        90	DUPLEX - ALL STYLES AND AGES
+       120	1-STORY PUD (Planned Unit Development) - 1946 & NEWER
+       150	1-1/2 STORY PUD - ALL AGES
+       160	2-STORY PUD - 1946 & NEWER
+       180	PUD - MULTILEVEL - INCL SPLIT LEV/FOYER
+       190	2 FAMILY CONVERSION - ALL STYLES AND AGES
+
+MSZoning: Identifies the general zoning classification of the sale.
+		
+       A	Agriculture
+       C	Commercial
+       FV	Floating Village Residential
+       I	Industrial
+       RH	Residential High Density
+       RL	Residential Low Density
+       RP	Residential Low Density Park 
+       RM	Residential Medium Density
+	
+LotFrontage: Linear feet of street connected to property
+
+LotArea: Lot size in square feet
+
+Street: Type of road access to property
+
+       Grvl	Gravel	
+       Pave	Paved
+       	
+Alley: Type of alley access to property
+
+       Grvl	Gravel
+       Pave	Paved
+       NA 	No alley access
+		
+LotShape: General shape of property
+
+       Reg	Regular	
+       IR1	Slightly irregular
+       IR2	Moderately Irregular
+       IR3	Irregular
+       
+LandContour: Flatness of the property
+
+       Lvl	Near Flat/Level	
+       Bnk	Banked - Quick and significant rise from street grade to building
+       HLS	Hillside - Significant slope from side to side
+       Low	Depression
+		
+Utilities: Type of utilities available
+		
+       AllPub	All public Utilities (E,G,W,& S)	
+       NoSewr	Electricity, Gas, and Water (Septic Tank)
+       NoSeWa	Electricity and Gas Only
+       ELO	Electricity only	
+	
+LotConfig: Lot configuration
+
+       Inside	Inside lot
+       Corner	Corner lot
+       CulDSac	Cul-de-sac
+       FR2	Frontage on 2 sides of property
+       FR3	Frontage on 3 sides of property
+	
+LandSlope: Slope of property
+		
+       Gtl	Gentle slope
+       Mod	Moderate Slope	
+       Sev	Severe Slope
+	
+Neighborhood: Physical locations within Ames city limits
+
+       Blmngtn	Bloomington Heights
+       Blueste	Bluestem
+       BrDale	Briardale
+       BrkSide	Brookside
+       ClearCr	Clear Creek
+       CollgCr	College Creek
+       Crawfor	Crawford
+       Edwards	Edwards
+       Gilbert	Gilbert
+       IDOTRR	Iowa DOT and Rail Road
+       MeadowV	Meadow Village
+       Mitchel	Mitchell
+       Names	North Ames
+       NoRidge	Northridge
+       NPkVill	Northpark Villa
+       NridgHt	Northridge Heights
+       NWAmes	Northwest Ames
+       OldTown	Old Town
+       SWISU	South & West of Iowa State University
+       Sawyer	Sawyer
+       SawyerW	Sawyer West
+       Somerst	Somerset
+       StoneBr	Stone Brook
+       Timber	Timberland
+       Veenker	Veenker
+			
+Condition1: Proximity to various conditions
+	
+       Artery	Adjacent to arterial street
+       Feedr	Adjacent to feeder street	
+       Norm	Normal	
+       RRNn	Within 200' of North-South Railroad
+       RRAn	Adjacent to North-South Railroad
+       PosN	Near positive off-site feature--park, greenbelt, etc.
+       PosA	Adjacent to postive off-site feature
+       RRNe	Within 200' of East-West Railroad
+       RRAe	Adjacent to East-West Railroad
+	
+Condition2: Proximity to various conditions (if more than one is present)
+		
+       Artery	Adjacent to arterial street
+       Feedr	Adjacent to feeder street	
+       Norm	Normal	
+       RRNn	Within 200' of North-South Railroad
+       RRAn	Adjacent to North-South Railroad
+       PosN	Near positive off-site feature--park, greenbelt, etc.
+       PosA	Adjacent to postive off-site feature
+       RRNe	Within 200' of East-West Railroad
+       RRAe	Adjacent to East-West Railroad
+	
+BldgType: Type of dwelling
+		
+       1Fam	Single-family Detached	
+       2FmCon	Two-family Conversion; originally built as one-family dwelling
+       Duplx	Duplex
+       TwnhsE	Townhouse End Unit
+       TwnhsI	Townhouse Inside Unit
+	
+HouseStyle: Style of dwelling
+	
+       1Story	One story
+       1.5Fin	One and one-half story: 2nd level finished
+       1.5Unf	One and one-half story: 2nd level unfinished
+       2Story	Two story
+       2.5Fin	Two and one-half story: 2nd level finished
+       2.5Unf	Two and one-half story: 2nd level unfinished
+       SFoyer	Split Foyer
+       SLvl	Split Level
+	
+OverallQual: Rates the overall material and finish of the house
+
+       10	Very Excellent
+       9	Excellent
+       8	Very Good
+       7	Good
+       6	Above Average
+       5	Average
+       4	Below Average
+       3	Fair
+       2	Poor
+       1	Very Poor
+	
+OverallCond: Rates the overall condition of the house
+
+       10	Very Excellent
+       9	Excellent
+       8	Very Good
+       7	Good
+       6	Above Average	
+       5	Average
+       4	Below Average	
+       3	Fair
+       2	Poor
+       1	Very Poor
+		
+YearBuilt: Original construction date
+
+YearRemodAdd: Remodel date (same as construction date if no remodeling or additions)
+
+RoofStyle: Type of roof
+
+       Flat	Flat
+       Gable	Gable
+       Gambrel	Gabrel (Barn)
+       Hip	Hip
+       Mansard	Mansard
+       Shed	Shed
+		
+RoofMatl: Roof material
+
+       ClyTile	Clay or Tile
+       CompShg	Standard (Composite) Shingle
+       Membran	Membrane
+       Metal	Metal
+       Roll	Roll
+       Tar&Grv	Gravel & Tar
+       WdShake	Wood Shakes
+       WdShngl	Wood Shingles
+		
+Exterior1st: Exterior covering on house
+
+       AsbShng	Asbestos Shingles
+       AsphShn	Asphalt Shingles
+       BrkComm	Brick Common
+       BrkFace	Brick Face
+       CBlock	Cinder Block
+       CemntBd	Cement Board
+       HdBoard	Hard Board
+       ImStucc	Imitation Stucco
+       MetalSd	Metal Siding
+       Other	Other
+       Plywood	Plywood
+       PreCast	PreCast	
+       Stone	Stone
+       Stucco	Stucco
+       VinylSd	Vinyl Siding
+       Wd Sdng	Wood Siding
+       WdShing	Wood Shingles
+	
+Exterior2nd: Exterior covering on house (if more than one material)
+
+       AsbShng	Asbestos Shingles
+       AsphShn	Asphalt Shingles
+       BrkComm	Brick Common
+       BrkFace	Brick Face
+       CBlock	Cinder Block
+       CemntBd	Cement Board
+       HdBoard	Hard Board
+       ImStucc	Imitation Stucco
+       MetalSd	Metal Siding
+       Other	Other
+       Plywood	Plywood
+       PreCast	PreCast
+       Stone	Stone
+       Stucco	Stucco
+       VinylSd	Vinyl Siding
+       Wd Sdng	Wood Siding
+       WdShing	Wood Shingles
+	
+MasVnrType: Masonry veneer type
+
+       BrkCmn	Brick Common
+       BrkFace	Brick Face
+       CBlock	Cinder Block
+       None	None
+       Stone	Stone
+	
+MasVnrArea: Masonry veneer area in square feet
+
+ExterQual: Evaluates the quality of the material on the exterior 
+		
+       Ex	Excellent
+       Gd	Good
+       TA	Average/Typical
+       Fa	Fair
+       Po	Poor
+		
+ExterCond: Evaluates the present condition of the material on the exterior
+		
+       Ex	Excellent
+       Gd	Good
+       TA	Average/Typical
+       Fa	Fair
+       Po	Poor
+		
+Foundation: Type of foundation
+		
+       BrkTil	Brick & Tile
+       CBlock	Cinder Block
+       PConc	Poured Contrete	
+       Slab	Slab
+       Stone	Stone
+       Wood	Wood
+		
+BsmtQual: Evaluates the height of the basement
+
+       Ex	Excellent (100+ inches)	
+       Gd	Good (90-99 inches)
+       TA	Typical (80-89 inches)
+       Fa	Fair (70-79 inches)
+       Po	Poor (<70 inches
+       NA	No Basement
+		
+BsmtCond: Evaluates the general condition of the basement
+
+       Ex	Excellent
+       Gd	Good
+       TA	Typical - slight dampness allowed
+       Fa	Fair - dampness or some cracking or settling
+       Po	Poor - Severe cracking, settling, or wetness
+       NA	No Basement
+	
+BsmtExposure: Refers to walkout or garden level walls
+
+       Gd	Good Exposure
+       Av	Average Exposure (split levels or foyers typically score average or above)	
+       Mn	Mimimum Exposure
+       No	No Exposure
+       NA	No Basement
+	
+BsmtFinType1: Rating of basement finished area
+
+       GLQ	Good Living Quarters
+       ALQ	Average Living Quarters
+       BLQ	Below Average Living Quarters	
+       Rec	Average Rec Room
+       LwQ	Low Quality
+       Unf	Unfinshed
+       NA	No Basement
+		
+BsmtFinSF1: Type 1 finished square feet
+
+BsmtFinType2: Rating of basement finished area (if multiple types)
+
+       GLQ	Good Living Quarters
+       ALQ	Average Living Quarters
+       BLQ	Below Average Living Quarters	
+       Rec	Average Rec Room
+       LwQ	Low Quality
+       Unf	Unfinshed
+       NA	No Basement
+
+BsmtFinSF2: Type 2 finished square feet
+
+BsmtUnfSF: Unfinished square feet of basement area
+
+TotalBsmtSF: Total square feet of basement area
+
+Heating: Type of heating
+		
+       Floor	Floor Furnace
+       GasA	Gas forced warm air furnace
+       GasW	Gas hot water or steam heat
+       Grav	Gravity furnace	
+       OthW	Hot water or steam heat other than gas
+       Wall	Wall furnace
+		
+HeatingQC: Heating quality and condition
+
+       Ex	Excellent
+       Gd	Good
+       TA	Average/Typical
+       Fa	Fair
+       Po	Poor
+		
+CentralAir: Central air conditioning
+
+       N	No
+       Y	Yes
+		
+Electrical: Electrical system
+
+       SBrkr	Standard Circuit Breakers & Romex
+       FuseA	Fuse Box over 60 AMP and all Romex wiring (Average)	
+       FuseF	60 AMP Fuse Box and mostly Romex wiring (Fair)
+       FuseP	60 AMP Fuse Box and mostly knob & tube wiring (poor)
+       Mix	Mixed
+		
+1stFlrSF: First Floor square feet
+ 
+2ndFlrSF: Second floor square feet
+
+LowQualFinSF: Low quality finished square feet (all floors)
+
+GrLivArea: Above grade (ground) living area square feet
+
+BsmtFullBath: Basement full bathrooms
+
+BsmtHalfBath: Basement half bathrooms
+
+FullBath: Full bathrooms above grade
+
+HalfBath: Half baths above grade
+
+Bedroom: Bedrooms above grade (does NOT include basement bedrooms)
+
+Kitchen: Kitchens above grade
+
+KitchenQual: Kitchen quality
+
+       Ex	Excellent
+       Gd	Good
+       TA	Typical/Average
+       Fa	Fair
+       Po	Poor
+       	
+TotRmsAbvGrd: Total rooms above grade (does not include bathrooms)
+
+Functional: Home functionality (Assume typical unless deductions are warranted)
+
+       Typ	Typical Functionality
+       Min1	Minor Deductions 1
+       Min2	Minor Deductions 2
+       Mod	Moderate Deductions
+       Maj1	Major Deductions 1
+       Maj2	Major Deductions 2
+       Sev	Severely Damaged
+       Sal	Salvage only
+		
+Fireplaces: Number of fireplaces
+
+FireplaceQu: Fireplace quality
+
+       Ex	Excellent - Exceptional Masonry Fireplace
+       Gd	Good - Masonry Fireplace in main level
+       TA	Average - Prefabricated Fireplace in main living area or Masonry Fireplace in basement
+       Fa	Fair - Prefabricated Fireplace in basement
+       Po	Poor - Ben Franklin Stove
+       NA	No Fireplace
+		
+GarageType: Garage location
+		
+       2Types	More than one type of garage
+       Attchd	Attached to home
+       Basment	Basement Garage
+       BuiltIn	Built-In (Garage part of house - typically has room above garage)
+       CarPort	Car Port
+       Detchd	Detached from home
+       NA	No Garage
+		
+GarageYrBlt: Year garage was built
+		
+GarageFinish: Interior finish of the garage
+
+       Fin	Finished
+       RFn	Rough Finished	
+       Unf	Unfinished
+       NA	No Garage
+		
+GarageCars: Size of garage in car capacity
+
+GarageArea: Size of garage in square feet
+
+GarageQual: Garage quality
+
+       Ex	Excellent
+       Gd	Good
+       TA	Typical/Average
+       Fa	Fair
+       Po	Poor
+       NA	No Garage
+		
+GarageCond: Garage condition
+
+       Ex	Excellent
+       Gd	Good
+       TA	Typical/Average
+       Fa	Fair
+       Po	Poor
+       NA	No Garage
+		
+PavedDrive: Paved driveway
+
+       Y	Paved 
+       P	Partial Pavement
+       N	Dirt/Gravel
+		
+WoodDeckSF: Wood deck area in square feet
+
+OpenPorchSF: Open porch area in square feet
+
+EnclosedPorch: Enclosed porch area in square feet
+
+3SsnPorch: Three season porch area in square feet
+
+ScreenPorch: Screen porch area in square feet
+
+PoolArea: Pool area in square feet
+
+PoolQC: Pool quality
+		
+       Ex	Excellent
+       Gd	Good
+       TA	Average/Typical
+       Fa	Fair
+       NA	No Pool
+		
+Fence: Fence quality
+		
+       GdPrv	Good Privacy
+       MnPrv	Minimum Privacy
+       GdWo	Good Wood
+       MnWw	Minimum Wood/Wire
+       NA	No Fence
+	
+MiscFeature: Miscellaneous feature not covered in other categories
+		
+       Elev	Elevator
+       Gar2	2nd Garage (if not described in garage section)
+       Othr	Other
+       Shed	Shed (over 100 SF)
+       TenC	Tennis Court
+       NA	None
+		
+MiscVal: $Value of miscellaneous feature
+
+MoSold: Month Sold (MM)
+
+YrSold: Year Sold (YYYY)
+
+SaleType: Type of sale
+		
+       WD 	Warranty Deed - Conventional
+       CWD	Warranty Deed - Cash
+       VWD	Warranty Deed - VA Loan
+       New	Home just constructed and sold
+       COD	Court Officer Deed/Estate
+       Con	Contract 15% Down payment regular terms
+       ConLw	Contract Low Down payment and low interest
+       ConLI	Contract Low Interest
+       ConLD	Contract Low Down
+       Oth	Other
+		
+SaleCondition: Condition of sale
+
+       Normal	Normal Sale
+       Abnorml	Abnormal Sale -  trade, foreclosure, short sale
+       AdjLand	Adjoining Land Purchase
+       Alloca	Allocation - two linked properties with separate deeds, typically condo with a garage unit	
+       Family	Sale between family members
+       Partial	Home was not completed when last assessed (associated with New Homes)

aide/example_tasks/house_prices/sample_submission.csv

@@ -0,0 +1,1460 @@
+Id,SalePrice
+1461,169277.0524984
+1462,187758.393988768
+1463,183583.683569555
+1464,179317.47751083
+1465,150730.079976501
+1466,177150.989247307
+1467,172070.659229164
+1468,175110.956519547
+1469,162011.698831665
+1470,160726.247831419
+1471,157933.279456005
+1472,145291.245020389
+1473,159672.017631819
+1474,164167.518301885
+1475,150891.638244053
+1476,179460.96518734
+1477,185034.62891405
+1478,182352.192644656
+1479,183053.458213802
+1480,187823.339254278
+1481,186544.114327568
+1482,158230.77520516
+1483,190552.829321091
+1484,147183.67487199
+1485,185855.300905493
+1486,174350.470676986
+1487,201740.620690863
+1488,162986.378895754
+1489,162330.199085679
+1490,165845.938616539
+1491,180929.622876974
+1492,163481.501519718
+1493,187798.076714233
+1494,198822.198942566
+1495,194868.409899858
+1496,152605.298564403
+1497,147797.702836811
+1498,150521.96899297
+1499,146991.630153739
+1500,150306.307814534
+1501,151164.372534604
+1502,151133.706960953
+1503,156214.042540726
+1504,171992.760735142
+1505,173214.912549738
+1506,192429.187345783
+1507,190878.69508543
+1508,194542.544135519
+1509,191849.439072822
+1510,176363.773907793
+1511,176954.185412429
+1512,176521.216975696
+1513,179436.704810176
+1514,220079.756777048
+1515,175502.918109444
+1516,188321.073833569
+1517,163276.324450004
+1518,185911.366293097
+1519,171392.830997252
+1520,174418.207020775
+1521,179682.709603774
+1522,179423.751581665
+1523,171756.918091777
+1524,166849.638174419
+1525,181122.168676666
+1526,170934.462746566
+1527,159738.292580329
+1528,174445.759557658
+1529,174706.363659627
+1530,164507.672539365
+1531,163602.512172832
+1532,154126.270249525
+1533,171104.853481351
+1534,167735.39270528
+1535,183003.613338104
+1536,172580.381161499
+1537,165407.889104689
+1538,176363.773907793
+1539,175182.950898522
+1540,190757.177789246
+1541,167186.995771991
+1542,167839.376779276
+1543,173912.421165137
+1544,154034.917445551
+1545,156002.955794336
+1546,168173.94329857
+1547,168882.437104132
+1548,168173.94329857
+1549,157580.177551642
+1550,181922.15256011
+1551,155134.227842592
+1552,188885.573319552
+1553,183963.193012381
+1554,161298.762306335
+1555,188613.66763056
+1556,175080.111822945
+1557,174744.400305232
+1558,168175.911336919
+1559,182333.472575006
+1560,158307.206742274
+1561,193053.055502348
+1562,175031.089987177
+1563,160713.294602908
+1564,173186.215014436
+1565,191736.7598055
+1566,170401.630997116
+1567,164626.577880222
+1568,205469.409444832
+1569,209561.784211885
+1570,182271.503072356
+1571,178081.549427793
+1572,178425.956138831
+1573,162015.318511503
+1574,181722.420373045
+1575,156705.730169433
+1576,182902.420342386
+1577,157574.595395085
+1578,184380.739100813
+1579,169364.469225677
+1580,175846.179822063
+1581,189673.295302136
+1582,174401.317715566
+1583,179021.448718583
+1584,189196.845337149
+1585,139647.095720655
+1586,161468.198288911
+1587,171557.32317862
+1588,179447.36804185
+1589,169611.619017694
+1590,172088.872655744
+1591,171190.624128768
+1592,154850.508361878
+1593,158617.655719941
+1594,209258.33693701
+1595,177939.027626751
+1596,194631.100299584
+1597,213618.871562568
+1598,198342.504228533
+1599,138607.971472497
+1600,150778.958976731
+1601,146966.230339786
+1602,162182.59620952
+1603,176825.940961269
+1604,152799.812402444
+1605,180322.322067129
+1606,177508.027228367
+1607,208029.642652019
+1608,181987.282510201
+1609,160172.72797397
+1610,176761.317654248
+1611,176515.497545231
+1612,176270.453065471
+1613,183050.846258475
+1614,150011.102062216
+1615,159270.537808667
+1616,163419.663729346
+1617,163399.983345859
+1618,173364.161505756
+1619,169556.835902417
+1620,183690.595995738
+1621,176980.914909382
+1622,204773.36222471
+1623,174728.655998442
+1624,181873.458244461
+1625,177322.000823979
+1626,193927.939041863
+1627,181715.622732304
+1628,199270.841200324
+1629,177109.589956218
+1630,153909.578271486
+1631,162931.203336223
+1632,166386.7567182
+1633,173719.30379824
+1634,179757.925656704
+1635,179007.601964376
+1636,180370.808623106
+1637,185102.616730563
+1638,198825.563452058
+1639,184294.576009142
+1640,200443.7920562
+1641,181294.784484153
+1642,174354.336267919
+1643,172023.677781517
+1644,181666.922855025
+1645,179024.491269586
+1646,178324.191575907
+1647,184534.676687694
+1648,159397.250378784
+1649,178430.966728182
+1650,177743.799385967
+1651,179395.305519087
+1652,151713.38474815
+1653,151713.38474815
+1654,168434.977996215
+1655,153999.100311019
+1656,164096.097354123
+1657,166335.403036551
+1658,163020.725375757
+1659,155862.510668829
+1660,182760.651095509
+1661,201912.270622883
+1662,185988.233987516
+1663,183778.44888032
+1664,170935.85921771
+1665,184468.908382254
+1666,191569.089663229
+1667,232991.025583822
+1668,180980.721388278
+1669,164279.13048219
+1670,183859.460411109
+1671,185922.465682076
+1672,191742.778119363
+1673,199954.072465842
+1674,180690.274752587
+1675,163099.3096358
+1676,140791.922472443
+1677,166481.86647592
+1678,172080.434496773
+1679,191719.161659178
+1680,160741.098612515
+1681,157829.546854733
+1682,196896.748596341
+1683,159675.423990355
+1684,182084.790901946
+1685,179233.926374487
+1686,155774.270901623
+1687,181354.326716058
+1688,179605.563663918
+1689,181609.34866147
+1690,178221.531623281
+1691,175559.920735795
+1692,200328.822792041
+1693,178630.060559899
+1694,177174.535221728
+1695,172515.687368714
+1696,204032.992922943
+1697,176023.232787689
+1698,202202.073341595
+1699,181734.480075862
+1700,183982.158993126
+1701,188007.94241481
+1702,185922.966763517
+1703,183978.544874918
+1704,177199.618638821
+1705,181878.647956764
+1706,173622.088728263
+1707,180728.168562655
+1708,176477.026606328
+1709,184282.266697609
+1710,162062.47538448
+1711,182550.070992189
+1712,180987.949624695
+1713,178173.79762147
+1714,179980.635948606
+1715,173257.637826205
+1716,177271.291059307
+1717,175338.355442312
+1718,177548.140549508
+1719,175969.91662932
+1720,175011.481953462
+1721,185199.372568143
+1722,188514.050228937
+1723,185080.145268797
+1724,157304.402574096
+1725,194260.859481297
+1726,181262.329995106
+1727,157003.292706732
+1728,182924.499359899
+1729,181902.586375439
+1730,188985.371708134
+1731,185290.904495068
+1732,177304.425752748
+1733,166274.900490809
+1734,177807.420530107
+1735,180330.624816201
+1736,179069.112234629
+1737,175943.371816948
+1738,185199.050609653
+1739,167350.910824524
+1740,149315.311876449
+1741,139010.847766793
+1742,155412.151845447
+1743,171308.313985441
+1744,176220.543265638
+1745,177643.434991809
+1746,187222.653264601
+1747,185635.132083154
+1748,206492.534215854
+1749,181681.021081956
+1750,180500.198072685
+1751,206486.17086841
+1752,161334.301195429
+1753,176156.558313965
+1754,191642.223478994
+1755,191945.808027777
+1756,164146.306037354
+1757,179883.057071096
+1758,178071.137668844
+1759,188241.637896875
+1760,174559.656173171
+1761,182347.363042264
+1762,191507.251872857
+1763,199751.865597358
+1764,162106.416145131
+1765,164575.982314367
+1766,179176.352180931
+1767,177327.403857584
+1768,177818.083761781
+1769,186965.204048443
+1770,178762.742169197
+1771,183322.866146283
+1772,178903.295931891
+1773,186570.129421778
+1774,199144.242829024
+1775,172154.713310956
+1776,177444.019201603
+1777,166200.938073485
+1778,158995.770555632
+1779,168273.282454755
+1780,189680.453052788
+1781,181681.021081956
+1782,160277.142643643
+1783,197318.54715833
+1784,162228.935604196
+1785,187340.455456083
+1786,181065.347037275
+1787,190233.609102705
+1788,157929.594852031
+1789,168557.001935469
+1790,160805.584645628
+1791,221648.391978216
+1792,180539.88079815
+1793,182105.616283853
+1794,166380.852603154
+1795,178942.155617426
+1796,162804.747800461
+1797,183077.684392615
+1798,171728.4720292
+1799,164786.741540638
+1800,177427.267170302
+1801,197318.54715833
+1802,178658.114178223
+1803,185437.320523764
+1804,169759.652489529
+1805,173986.635055186
+1806,168607.664289468
+1807,194138.519145183
+1808,192502.440921994
+1809,176746.969818601
+1810,177604.891703134
+1811,193283.746584832
+1812,181627.061006609
+1813,169071.62025834
+1814,167398.006470987
+1815,150106.505141704
+1816,159650.304285848
+1817,179471.23597476
+1818,177109.589956218
+1819,166558.113328453
+1820,153796.714319583
+1821,174520.152570658
+1822,196297.95829524
+1823,169100.681601175
+1824,176911.319164431
+1825,169234.6454828
+1826,172386.297919134
+1827,156031.904802362
+1828,168202.892306596
+1829,166505.984017547
+1830,176507.37022149
+1831,180116.752553161
+1832,183072.740591406
+1833,189595.964677698
+1834,167523.919076265
+1835,210817.775863413
+1836,172942.930813351
+1837,145286.278144089
+1838,176468.653371492
+1839,159040.069562187
+1840,178518.204332507
+1841,169163.980786825
+1842,189786.685274579
+1843,181246.728523853
+1844,176349.927153587
+1845,205266.631009142
+1846,187397.993362224
+1847,208943.427726113
+1848,165014.532907657
+1849,182492.037566236
+1850,161718.71259042
+1851,180084.118941162
+1852,178534.950802179
+1853,151217.259961305
+1854,156342.717587562
+1855,188511.443835239
+1856,183570.337896789
+1857,225810.160292177
+1858,214217.401131694
+1859,187665.64101603
+1860,161157.177744039
+1861,187643.992594193
+1862,228156.372839158
+1863,220449.534665317
+1864,220522.352084222
+1865,156647.763531624
+1866,187388.833374873
+1867,178640.723791573
+1868,180847.216739049
+1869,159505.170529478
+1870,164305.538020654
+1871,180181.19673723
+1872,184602.734989972
+1873,193440.372174434
+1874,184199.788209911
+1875,196241.892907637
+1876,175588.618271096
+1877,179503.046546829
+1878,183658.076582555
+1879,193700.976276404
+1880,165399.62450704
+1881,186847.944787446
+1882,198127.73287817
+1883,183320.898107934
+1884,181613.606696657
+1885,178298.791761954
+1886,185733.534000593
+1887,180008.188485489
+1888,175127.59621604
+1889,183467.176862723
+1890,182705.546021743
+1891,152324.943593181
+1892,169878.515981342
+1893,183735.975076576
+1894,224118.280105941
+1895,169355.202465146
+1896,180054.276407441
+1897,174081.601977368
+1898,168494.985022146
+1899,181871.598843299
+1900,173554.489658383
+1901,169805.382165577
+1902,176192.990728755
+1903,204264.39284654
+1904,169630.906956928
+1905,185724.838807268
+1906,195699.036281861
+1907,189494.276162169
+1908,149607.905673439
+1909,154650.199045978
+1910,151579.558140433
+1911,185147.380531144
+1912,196314.53120359
+1913,210802.395364155
+1914,166271.2863726
+1915,154865.359142973
+1916,173575.5052865
+1917,179399.563554274
+1918,164280.776562049
+1919,171247.48948121
+1920,166878.587182445
+1921,188129.459710994
+1922,183517.34369691
+1923,175522.026925727
+1924,190060.105331152
+1925,174179.824771856
+1926,171059.523675194
+1927,183004.186769318
+1928,183601.647387418
+1929,163539.327185998
+1930,164677.676391525
+1931,162395.073865424
+1932,182207.6323195
+1933,192223.939790304
+1934,176391.829390125
+1935,181913.179121348
+1936,179136.097888261
+1937,196595.568243212
+1938,194822.365690957
+1939,148356.669440918
+1940,160387.604263899
+1941,181276.500571809
+1942,192474.817899346
+1943,157699.907796437
+1944,215785.540813051
+1945,181824.300998793
+1946,221813.00948166
+1947,165281.292597397
+1948,255629.49047034
+1949,173154.590990955
+1950,183884.65246539
+1951,200210.353608489
+1952,186599.221265342
+1953,192718.532696106
+1954,178628.665952764
+1955,180650.342418406
+1956,206003.107947263
+1957,166457.67844853
+1958,202916.221653487
+1959,192463.969983091
+1960,171775.497189898
+1961,175249.222149411
+1962,147086.59893993
+1963,149709.672100371
+1964,171411.404533743
+1965,178188.964799425
+1966,156491.711373235
+1967,180953.241201168
+1968,203909.759061135
+1969,175470.149087545
+1970,205578.333622415
+1971,199428.857699441
+1972,187599.163869476
+1973,192265.198109864
+1974,196666.554897677
+1975,155537.862252682
+1976,169543.240620935
+1977,202487.010170501
+1978,208232.716273485
+1979,173621.195202569
+1980,172414.608571812
+1981,164400.75641556
+1982,160480.424024781
+1983,156060.853810389
+1984,157437.192820581
+1985,158163.720929772
+1986,154849.043268978
+1987,152186.609341561
+1988,180340.215399228
+1989,178344.62451356
+1990,190170.382266827
+1991,168092.975480832
+1992,178757.912566805
+1993,174518.256882082
+1994,198168.490116289
+1995,176882.693978902
+1996,183801.672896251
+1997,196400.046680661
+1998,172281.605004025
+1999,196380.366297173
+2000,198228.354306682
+2001,195556.581268962
+2002,186453.264469043
+2003,181869.381196234
+2004,175610.840124147
+2005,183438.730800145
+2006,179584.488673295
+2007,182386.152242034
+2008,160750.367237054
+2009,182477.505046008
+2010,187720.359207171
+2011,187201.942081511
+2012,176385.102235149
+2013,175901.787841278
+2014,182584.280198283
+2015,195664.686104237
+2016,181420.346494222
+2017,176676.04995228
+2018,181594.678867334
+2019,178521.747964951
+2020,175895.883726231
+2021,168468.005916477
+2022,200973.129447888
+2023,197030.641992202
+2024,192867.417844592
+2025,196449.247639381
+2026,141684.196398607
+2027,153353.334123901
+2028,151143.549016705
+2029,163753.087114229
+2030,158682.460013921
+2031,144959.835250915
+2032,160144.390548579
+2033,156286.534303521
+2034,165726.707619571
+2035,182427.481047359
+2036,173310.56154032
+2037,173310.56154032
+2038,151556.01403002
+2039,158908.146068683
+2040,209834.383092536
+2041,192410.516550815
+2042,174026.247294886
+2043,195499.830115336
+2044,200918.018812493
+2045,207243.616023976
+2046,196149.783851876
+2047,192097.914850217
+2048,178570.948923671
+2049,228617.968325428
+2050,199929.884438451
+2051,160206.365612859
+2052,179854.431885567
+2053,185987.340461822
+2054,161122.505607926
+2055,175949.342720138
+2056,183683.590595324
+2057,176401.34762338
+2058,205832.532527897
+2059,177799.799849436
+2060,167565.362080406
+2061,186348.958436557
+2062,179782.759465081
+2063,169837.623333323
+2064,178817.275675758
+2065,174444.479149339
+2066,192834.968917174
+2067,196564.717984981
+2068,206977.567039357
+2069,157054.253944128
+2070,175142.948078577
+2071,159932.1643654
+2072,182801.408333628
+2073,181510.375176825
+2074,181613.035129451
+2075,186920.512597635
+2076,157950.170625222
+2077,176115.159022876
+2078,182744.514344465
+2079,180660.683691591
+2080,160775.629777099
+2081,186711.715848082
+2082,223581.758190888
+2083,172330.943236652
+2084,163474.633393212
+2085,175308.263299874
+2086,187462.725306432
+2087,180655.101535034
+2088,152121.98603454
+2089,159856.233909727
+2090,186559.854936737
+2091,183962.550959411
+2092,162107.168699296
+2093,162582.288981283
+2094,154407.701597409
+2095,181625.666399474
+2096,164810.609473548
+2097,176429.401241704
+2098,179188.089925259
+2099,145997.635377703
+2100,218676.768270367
+2101,188323.861214226
+2102,168690.0722914
+2103,165088.746797705
+2104,191435.007885166
+2105,168864.404664512
+2106,176041.882371574
+2107,215911.674390325
+2108,167388.238629016
+2109,163854.786753017
+2110,163299.477980171
+2111,178298.214633119
+2112,176376.586164775
+2113,170211.043976522
+2114,170818.344786366
+2115,174388.867432503
+2116,161112.987374671
+2117,172179.082325307
+2118,157798.309713876
+2119,169106.151422924
+2120,170129.531364292
+2121,157680.227412949
+2122,162690.209131977
+2123,146968.379365095
+2124,181507.721372455
+2125,191215.589752983
+2126,189432.689844522
+2127,207271.484957719
+2128,170030.807488363
+2129,148409.806476335
+2130,193850.613979055
+2131,193808.319298263
+2132,166300.235380627
+2133,163474.633393212
+2134,177473.606564978
+2135,157443.925537187
+2136,180681.007992057
+2137,183463.17030026
+2138,182481.763081195
+2139,193717.15117887
+2140,182782.55099007
+2141,175530.651633287
+2142,177804.057884623
+2143,159448.670848577
+2144,181338.976717529
+2145,178553.558537021
+2146,162820.928264556
+2147,188832.479997186
+2148,164682.185899437
+2149,181549.735943801
+2150,199158.097008868
+2151,152889.520990566
+2152,181150.551679116
+2153,181416.732376013
+2154,164391.238182305
+2155,185421.046498812
+2156,193981.327550004
+2157,178824.324789223
+2158,209270.051606246
+2159,177801.266806344
+2160,179053.762236101
+2161,178762.170601992
+2162,184655.300458183
+2163,191284.655779772
+2164,179598.085818785
+2165,167517.628078595
+2166,182873.903794044
+2167,177484.91371363
+2168,188444.597319524
+2169,179184.153848562
+2170,184365.175780982
+2171,184479.322005212
+2172,182927.863869391
+2173,178611.639373646
+2174,181943.343613558
+2175,175080.614768394
+2176,190720.794649138
+2177,198422.868144723
+2178,184482.11308349
+2179,139214.952187861
+2180,169233.113601757
+2181,180664.118686848
+2182,178818.742632666
+2183,180422.049969947
+2184,178601.93645581
+2185,183083.159775993
+2186,173163.101499699
+2187,185968.161159774
+2188,171226.050683054
+2189,281643.976116786
+2190,160031.711281258
+2191,162775.979779394
+2192,160735.445970193
+2193,166646.109048572
+2194,188384.548444549
+2195,165830.697255197
+2196,182138.358533039
+2197,171595.397975647
+2198,160337.079183809
+2199,191215.088671543
+2200,166956.093232213
+2201,186581.830878692
+2202,176450.548582099
+2203,193743.194909801
+2204,198882.566078408
+2205,176385.102235149
+2206,162447.639333636
+2207,193782.555676777
+2208,183653.890897141
+2209,210578.623546866
+2210,158527.164107319
+2211,163081.025723456
+2212,174388.867432503
+2213,191905.870131966
+2214,174388.867432503
+2215,161642.711648983
+2216,186939.507215101
+2217,172482.165792649
+2218,159695.999763546
+2219,157230.369671007
+2220,179188.089925259
+2221,157972.82120994
+2222,156804.951429181
+2223,211491.972463654
+2224,186537.246201062
+2225,200468.161070551
+2226,182241.340444154
+2227,157342.225898399
+2228,182022.387105998
+2229,181244.510876788
+2230,178556.671573788
+2231,189547.199876284
+2232,187948.65165563
+2233,194107.287565956
+2234,183521.710369283
+2235,183682.123638416
+2236,178483.353073443
+2237,184003.879764736
+2238,171318.59033449
+2239,162039.754313997
+2240,154846.252190699
+2241,194822.365690957
+2242,169788.738771463
+2243,178891.554489941
+2244,152084.772428865
+2245,139169.86642879
+2246,192439.536044606
+2247,161067.859766557
+2248,158762.648504781
+2249,175569.690441774
+2250,183659.795012187
+2251,280618.132617258
+2252,180051.809151659
+2253,176519.18031559
+2254,179028.429210291
+2255,177161.583857224
+2256,180081.508849842
+2257,205895.254584712
+2258,183389.78131415
+2259,178543.647859512
+2260,194798.320499104
+2261,162845.613675766
+2262,148103.867006579
+2263,201016.171121215
+2264,277936.12694354
+2265,249768.279823405
+2266,161596.052159825
+2267,158011.114889899
+2268,194089.683858004
+2269,181733.336941451
+2270,182852.32772198
+2271,189893.003058465
+2272,194650.210979875
+2273,187904.461286262
+2274,171774.925622692
+2275,177998.685921479
+2276,175648.484325498
+2277,196918.071362067
+2278,184299.838071218
+2279,182379.855682734
+2280,184050.725802482
+2281,158296.975970284
+2282,175053.355553278
+2283,162293.376090644
+2284,186328.880047186
+2285,151422.116936538
+2286,181969.358707768
+2287,189122.67702416
+2288,185645.475220346
+2289,182829.898109257
+2290,195848.788183328
+2291,198785.059550672
+2292,181676.126555428
+2293,194131.012663328
+2294,201416.004864508
+2295,185096.577205616
+2296,195158.972598372
+2297,184795.783735112
+2298,189168.263864671
+2299,216855.260149095
+2300,184946.642483576
+2301,189317.51282069
+2302,180803.277842406
+2303,175061.18585763
+2304,179074.839090732
+2305,145708.764336107
+2306,142398.022752011
+2307,161474.534863641
+2308,157025.945155458
+2309,163424.037827357
+2310,164692.778645345
+2311,152163.2443541
+2312,192383.215486656
+2313,182520.230322476
+2314,187254.507549722
+2315,176489.659740359
+2316,181520.466841293
+2317,186414.978214721
+2318,185197.764639705
+2319,178657.794083741
+2320,179731.198023759
+2321,161748.271317074
+2322,158608.749069322
+2323,178807.370559878
+2324,184187.158803897
+2325,181686.10402108
+2326,190311.050228337
+2327,192252.496354076
+2328,193954.849525775
+2329,181044.201560887
+2330,180258.131219792
+2331,199641.657313834
+2332,197530.775205517
+2333,191777.196949138
+2334,195779.543033588
+2335,202112.046522999
+2336,192343.34807661
+2337,185191.359443218
+2338,186760.207965688
+2339,177733.78193528
+2340,164430.391189608
+2341,185299.601552401
+2342,186414.012339254
+2343,176401.921054593
+2344,182381.322639642
+2345,176334.184710805
+2346,184901.735847457
+2347,180085.766885029
+2348,184901.735847457
+2349,183967.561548763
+2350,193046.301574659
+2351,168538.969495849
+2352,170157.842016969
+2353,196559.709259637
+2354,177133.709361852
+2355,181553.279576244
+2356,185770.606634739
+2357,177017.595099274
+2358,184123.358536806
+2359,165970.357492196
+2360,158151.985049452
+2361,177086.476441481
+2362,196373.896176551
+2363,172465.707083115
+2364,168590.782409896
+2365,158820.474171061
+2366,151611.37057651
+2367,152125.028585543
+2368,158404.073081048
+2369,160692.078640755
+2370,170175.22684199
+2371,169854.436591138
+2372,183410.785819008
+2373,180347.194026928
+2374,178930.528374292
+2375,153346.220086301
+2376,182675.204270589
+2377,180770.649792036
+2378,188714.148087543
+2379,191393.608594076
+2380,174016.157494425
+2381,183189.685319552
+2382,183621.508757866
+2383,168991.29635758
+2384,185306.650665866
+2385,189030.680303208
+2386,179208.665698449
+2387,174901.452792889
+2388,168337.406544343
+2389,158234.96461859
+2390,179562.453368834
+2391,174176.391640607
+2392,173931.531845427
+2393,184111.729429665
+2394,179374.482001188
+2395,207348.811884535
+2396,186983.419339031
+2397,206779.094049527
+2398,177472.074683935
+2399,156727.948324862
+2400,157090.568462479
+2401,160387.032696693
+2402,172410.28005086
+2403,191603.365657467
+2404,182152.207151253
+2405,180161.697340702
+2406,169652.235284283
+2407,182503.520140218
+2408,179714.630677039
+2409,180282.570719908
+2410,192600.338060371
+2411,166115.491248565
+2412,186379.553524443
+2413,184361.992258449
+2414,186220.965458121
+2415,198176.47090687
+2416,168437.776500131
+2417,178003.582312015
+2418,179180.469244588
+2419,191930.561104806
+2420,175590.266214964
+2421,176713.19307219
+2422,180159.090947005
+2423,188090.100808026
+2424,186184.717727913
+2425,223055.588672278
+2426,158270.753116401
+2427,184733.12846644
+2428,199926.378957429
+2429,175075.785166001
+2430,180917.925148076
+2431,182067.760625207
+2432,178238.60191545
+2433,173454.944606532
+2434,176821.936262814
+2435,183642.191304235
+2436,177254.582741058
+2437,168715.950111702
+2438,180096.931198144
+2439,160620.728178758
+2440,175286.544392273
+2441,153494.783276297
+2442,156407.65915545
+2443,162162.525245786
+2444,166809.886827197
+2445,172929.156408918
+2446,193514.330894137
+2447,181612.141603756
+2448,191745.386377068
+2449,171369.325038261
+2450,184425.470567051
+2451,170563.252355189
+2452,184522.369240168
+2453,164968.947931153
+2454,157939.621592364
+2455,151520.381580069
+2456,176129.508722531
+2457,171112.978971478
+2458,169762.081624282
+2459,162246.828936295
+2460,171339.303381589
+2461,189034.753653813
+2462,175758.873595981
+2463,163351.721489893
+2464,189806.546645026
+2465,175370.990918319
+2466,196895.599900301
+2467,176905.917994834
+2468,176866.557227858
+2469,163590.677170026
+2470,212693.502958393
+2471,192686.931747717
+2472,181578.684951827
+2473,166475.457581812
+2474,185998.255166219
+2475,185527.714877908
+2476,159027.118197683
+2477,181169.654933769
+2478,176732.915304722
+2479,191619.294648838
+2480,189114.303789324
+2481,180934.635330334
+2482,164573.372223048
+2483,173902.011270196
+2484,165625.127741229
+2485,179555.219570787
+2486,196899.720661579
+2487,207566.12470446
+2488,163899.981149274
+2489,189179.428177786
+2490,193892.880023125
+2491,178980.874331431
+2492,179749.876244365
+2493,197999.674975598
+2494,203717.470295797
+2495,185249.261156892
+2496,201691.208274848
+2497,181956.548314794
+2498,171895.936275806
+2499,187245.168439419
+2500,157816.77461318
+2501,191702.912573325
+2502,198599.420028908
+2503,187193.313676329
+2504,220514.993999535
+2505,181814.527595192
+2506,183750.755371907
+2507,183000.431679579
+2508,185830.971906573
+2509,185497.872344187
+2510,179613.437681321
+2511,164454.967963631
+2512,185127.237217638
+2513,178750.613844623
+2514,160927.61044889
+2515,192562.808057836
+2516,180990.24148554
+2517,180064.941503122
+2518,196070.997393789
+2519,180352.919019023
+2520,183367.953769362
+2521,176734.841494027
+2522,180848.220765939
+2523,187806.059368823
+2524,180521.52640004
+2525,181502.754496154
+2526,174525.87942676
+2527,188927.984063168
+2528,184728.870431253
+2529,179857.975518011
+2530,180962.868071609
+2531,179194.066390078
+2532,179591.789259484
+2533,180638.463702549
+2534,185846.215131922
+2535,195174.031139141
+2536,192474.56829063
+2537,164200.595496827
+2538,178403.094096818
+2539,170774.84018302
+2540,179879.945898337
+2541,177668.192752792
+2542,180174.328610725
+2543,170643.303572141
+2544,165448.004289838
+2545,195531.754886222
+2546,165314.177682121
+2547,172532.757660882
+2548,203310.218069877
+2549,175090.062515883
+2550,230841.338626282
+2551,155225.19006632
+2552,168322.342441945
+2553,165956.259265265
+2554,193956.817564124
+2555,171070.367893827
+2556,166285.243628001
+2557,182875.801346628
+2558,218108.536769738
+2559,174378.777632042
+2560,164731.316372391
+2561,156969.695083273
+2562,173388.854342604
+2563,177559.628685119
+2564,194297.789279905
+2565,174894.588364005
+2566,196544.144075798
+2567,179036.158528149
+2568,211423.986511149
+2569,208156.398935188
+2570,159233.941347257
+2571,210820.115134931
+2572,140196.10979821
+2573,198678.469082978
+2574,186818.610760803
+2575,175044.797633861
+2576,180031.162892704
+2577,176889.171525162
+2578,159638.856165666
+2579,154287.264375509
+2580,191885.618181273
+2581,177503.378612934
+2582,166548.31684976
+2583,164475.14942856
+2584,167484.744857879
+2585,188683.160555403
+2586,162243.399502668
+2587,180807.213919103
+2588,176279.079637039
+2589,163438.959094218
+2590,161495.5393685
+2591,216032.303722443
+2592,176632.181541401
+2593,168743.001567144
+2594,183810.11848086
+2595,156794.36054728
+2596,169136.43011395
+2597,183203.318752456
+2598,213252.926930889
+2599,190550.327866959
+2600,234707.209860273
+2601,135751.318892816
+2602,164228.45886894
+2603,153219.437030419
+2604,164210.746523801
+2605,163883.229117973
+2606,154892.776269956
+2607,197092.08733832
+2608,228148.376399122
+2609,178680.587503997
+2610,165643.341167808
+2611,222406.642660249
+2612,184021.843582599
+2613,170871.094939159
+2614,189562.873697309
+2615,170591.884966356
+2616,172934.351682851
+2617,186425.069879189
+2618,218648.131133006
+2619,183035.606761141
+2620,178378.906069427
+2621,184516.716597846
+2622,181419.5253183
+2623,196858.923438425
+2624,189228.701486278
+2625,208973.380761028
+2626,180269.86896412
+2627,159488.713683953
+2628,191490.299507521
+2629,228684.245137946
+2630,201842.998700429
+2631,209242.82289186
+2632,202357.62258493
+2633,168238.61218265
+2634,202524.12465369
+2635,170588.771929588
+2636,198375.31512987
+2637,170636.827889889
+2638,181991.079479377
+2639,183994.54251844
+2640,182951.482193584
+2641,174126.297156192
+2642,170575.496742588
+2643,175332.239869971
+2644,167522.061539111
+2645,168095.583738538
+2646,154406.415627461
+2647,170996.973346087
+2648,159056.890245639
+2649,181373.6165193
+2650,152272.560975937
+2651,168664.346821336
+2652,211007.008292301
+2653,182909.515032911
+2654,203926.829353303
+2655,179082.825442944
+2656,206260.099795032
+2657,181732.443415757
+2658,189698.740693148
+2659,203074.34678979
+2660,201670.634365666
+2661,173756.812589691
+2662,181387.076390881
+2663,184859.155270535
+2664,158313.615666777
+2665,151951.955409666
+2666,162537.52704471
+2667,178998.337067854
+2668,186732.584943041
+2669,187323.318406165
+2670,199437.232798284
+2671,185546.680858653
+2672,161595.015798593
+2673,154672.422763036
+2674,159355.710116165
+2675,155919.014077746
+2676,182424.87095604
+2677,178100.589622319
+2678,202577.900044456
+2679,177862.778940605
+2680,182056.024744887
+2681,191403.199177104
+2682,196264.754980043
+2683,209375.003419718
+2684,196691.81930173
+2685,192458.431539585
+2686,182242.80926507
+2687,183259.503900506
+2688,188108.243748841
+2689,171418.640195797
+2690,194698.882220432
+2691,174841.84007522
+2692,172965.476488899
+2693,189386.323677132
+2694,185682.618340257
+2695,176412.012719061
+2696,174976.489722867
+2697,180718.581707643
+2698,186131.188248242
+2699,165220.786354033
+2700,164115.893800435
+2701,182125.729127024
+2702,182285.140233276
+2703,196325.442210366
+2704,164865.215329881
+2705,182694.492209823
+2706,185425.485520958
+2707,171414.7041191
+2708,183433.472466085
+2709,176844.981155794
+2710,180568.187753206
+2711,185948.625475832
+2712,189388.291715481
+2713,142754.489165865
+2714,156106.800760811
+2715,155895.397617561
+2716,159851.977738548
+2717,185157.832305524
+2718,180716.291710805
+2719,176901.093954071
+2720,181017.222455218
+2721,183269.159407668
+2722,193550.830097069
+2723,170625.842699726
+2724,182012.405942725
+2725,179162.507290733
+2726,183269.159407668
+2727,180589.836175042
+2728,181465.935198741
+2729,196053.029878304
+2730,183421.020319014
+2731,167926.839083612
+2732,168027.530997889
+2733,182164.26685407
+2734,172469.071592608
+2735,181059.374300472
+2736,182997.570115536
+2737,166140.504179894
+2738,198515.546934075
+2739,193789.648503294
+2740,173550.025727531
+2741,176487.943174734
+2742,188813.302559147
+2743,178531.911979192
+2744,182145.731469001
+2745,179196.465024103
+2746,169618.349900686
+2747,170010.168655046
+2748,181739.671652174
+2749,172846.934955574
+2750,195560.8830172
+2751,180358.114292956
+2752,211817.702818093
+2753,176170.128686742
+2754,234492.248263699
+2755,182450.956536015
+2756,174902.068073146
+2757,173684.174293738
+2758,147196.673677562
+2759,175231.189709791
+2760,193417.64740633
+2761,183313.601249761
+2762,180882.250849082
+2763,186735.697979808
+2764,172922.865411247
+2765,202551.677190573
+2766,190485.634074173
+2767,173439.49362151
+2768,196613.598849219
+2769,178152.259700828
+2770,174519.904825949
+2771,172627.796932837
+2772,173732.689486435
+2773,209219.844787023
+2774,181059.374300472
+2775,188515.443002459
+2776,182164.26685407
+2777,188137.901597981
+2778,158893.54306269
+2779,189579.65066771
+2780,165229.803505847
+2781,162186.071220207
+2782,166374.879866351
+2783,161665.184974757
+2784,175079.328798445
+2785,203840.874021305
+2786,152129.078861057
+2787,181012.141380101
+2788,161305.53503837
+2789,203326.392972343
+2790,168385.571141831
+2791,183564.365159986
+2792,163784.619440861
+2793,171989.192193993
+2794,180839.95616829
+2795,170895.923185907
+2796,174071.054808518
+2797,259423.859147546
+2798,188000.824679588
+2799,179171.703565498
+2800,171022.241447762
+2801,174126.297156192
+2802,187625.573271948
+2803,199567.946369234
+2804,205328.078219268
+2805,166231.535025379
+2806,154743.91606057
+2807,159714.537012622
+2808,185563.069082422
+2809,171500.796725006
+2810,180983.443844799
+2811,183141.236914997
+2812,178498.634450214
+2813,224323.710512388
+2814,218200.642127877
+2815,182283.177756557
+2816,190054.639237419
+2817,160192.453934518
+2818,171289.393581756
+2819,151131.098733642
+2820,181721.458225594
+2821,172725.053851858
+2822,222438.699143414
+2823,235419.373448928
+2824,185150.926027596
+2825,184772.239624699
+2826,180658.216435809
+2827,209673.316647174
+2828,205939.810625621
+2829,165633.573325837
+2830,186030.317211014
+2831,160312.319589212
+2832,190702.440251029
+2833,175122.810326699
+2834,183783.13937519
+2835,178290.666302221
+2836,181605.343963015
+2837,187992.451444752
+2838,188885.11781517
+2839,189959.344795118
+2840,179258.619211334
+2841,181518.750275669
+2842,193008.659237315
+2843,186313.89385619
+2844,181499.39185067
+2845,174126.297156192
+2846,183918.612062767
+2847,184114.270899227
+2848,158540.947801398
+2849,197034.759055859
+2850,185170.284452595
+2851,221134.533635148
+2852,184306.637575967
+2853,199792.302740996
+2854,143237.803559736
+2855,177294.838897736
+2856,182368.620883855
+2857,176487.943174734
+2858,183849.408762071
+2859,184964.141507413
+2860,196395.969632434
+2861,188374.936650438
+2862,176261.296806135
+2863,163628.142248426
+2864,180618.032628904
+2865,161647.329794081
+2866,167129.598867773
+2867,174750.988352687
+2868,177560.202116333
+2869,192577.796112839
+2870,199202.898960871
+2871,182818.156667308
+2872,148217.262540651
+2873,188997.797082492
+2874,185807.928877601
+2875,177030.477842021
+2876,175942.474593632
+2877,172912.518576433
+2878,198359.248864591
+2879,184379.133036383
+2880,194255.566948886
+2881,209449.651603064
+2882,169979.323958443
+2883,188206.281858748
+2884,186412.438609167
+2885,196761.386409959
+2886,208353.269558209
+2887,166548.067241044
+2888,175942.474593632
+2889,166790.457916434
+2890,160515.850579067
+2891,192167.621096362
+2892,178751.551083369
+2893,198678.894117024
+2894,164553.120272354
+2895,156887.932862327
+2896,164185.777305524
+2897,212992.120630876
+2898,197468.550532521
+2899,180106.84373966
+2900,183972.071056674
+2901,245283.198337927
+2902,170351.963410756
+2903,195596.307707478
+2904,189369.756330412
+2905,223667.404551664
+2906,169335.310624364
+2907,167411.02835165
+2908,187709.555003968
+2909,196526.002998991
+2910,137402.569855589
+2911,165086.775061735
+2912,188506.431412274
+2913,172917.456816012
+2914,166274.325225982
+2915,167081.220948984
+2916,164788.778231138
+2917,219222.423400059
+2918,184924.279658997
+2919,187741.866657478

aide/interpreter.py

@@ -0,0 +1,311 @@
+"""
+Python interpreter for executing code snippets and capturing their output.
+Supports:
+- captures stdout and stderr
+- captures exceptions and stack traces
+- limits execution time
+"""
+
+import logging
+import os
+import queue
+import signal
+import sys
+import time
+import traceback
+from dataclasses import dataclass
+from multiprocessing import Process, Queue
+from pathlib import Path
+
+import humanize
+from dataclasses_json import DataClassJsonMixin
+
+logger = logging.getLogger("aide")
+
+
+@dataclass
+class ExecutionResult(DataClassJsonMixin):
+    """
+    Result of executing a code snippet in the interpreter.
+    Contains the output, execution time, and exception information.
+    """
+
+    term_out: list[str]
+    exec_time: float
+    exc_type: str | None
+    exc_info: dict | None = None
+    exc_stack: list[tuple] | None = None
+
+
+def exception_summary(e, working_dir, exec_file_name, format_tb_ipython):
+    """Generates a string that summarizes an exception and its stack trace (either in standard python repl or in IPython format)."""
+    if format_tb_ipython:
+        import IPython.core.ultratb
+
+        # tb_offset = 1 to skip parts of the stack trace in weflow code
+        tb = IPython.core.ultratb.VerboseTB(tb_offset=1, color_scheme="NoColor")
+        tb_str = str(tb.text(*sys.exc_info()))
+    else:
+        tb_lines = traceback.format_exception(e)
+        # skip parts of stack trace in weflow code
+        tb_str = "".join(
+            [
+                line
+                for line in tb_lines
+                if "aide/" not in line and "importlib" not in line
+            ]
+        )
+        # tb_str = "".join([l for l in tb_lines])
+
+    # replace whole path to file with just filename (to remove agent workspace dir)
+    tb_str = tb_str.replace(str(working_dir / exec_file_name), exec_file_name)
+
+    exc_info = {}
+    if hasattr(e, "args"):
+        exc_info["args"] = [str(i) for i in e.args]
+    for att in ["name", "msg", "obj"]:
+        if hasattr(e, att):
+            exc_info[att] = str(getattr(e, att))
+
+    tb = traceback.extract_tb(e.__traceback__)
+    exc_stack = [(t.filename, t.lineno, t.name, t.line) for t in tb]
+
+    return tb_str, e.__class__.__name__, exc_info, exc_stack
+
+
+class RedirectQueue:
+    def __init__(self, queue, timeout=5):
+        self.queue = queue
+        self.timeout = timeout
+
+    def write(self, msg):
+        try:
+            self.queue.put(msg, timeout=self.timeout)
+        except queue.Full:
+            logger.warning("Queue write timed out")
+
+    def flush(self):
+        pass
+
+
+class Interpreter:
+    def __init__(
+        self,
+        working_dir: Path | str,
+        timeout: int = 3600,
+        format_tb_ipython: bool = False,
+        agent_file_name: str = "runfile.py",
+    ):
+        """
+        Simulates a standalone Python REPL with an execution time limit.
+
+        Args:
+            working_dir (Path | str): working directory of the agent
+            timeout (int, optional): Timeout for each code execution step. Defaults to 3600.
+            format_tb_ipython (bool, optional): Whether to use IPython or default python REPL formatting for exceptions. Defaults to False.
+            agent_file_name (str, optional): The name for the agent's code file. Defaults to "runfile.py".
+        """
+        # this really needs to be a path, otherwise causes issues that don't raise exc
+        self.working_dir = Path(working_dir).resolve()
+        assert (
+            self.working_dir.exists()
+        ), f"Working directory {self.working_dir} does not exist"
+        self.timeout = timeout
+        self.format_tb_ipython = format_tb_ipython
+        self.agent_file_name = agent_file_name
+        self.process: Process = None  # type: ignore
+
+    def child_proc_setup(self, result_outq: Queue) -> None:
+        # disable all warnings (before importing anything)
+        import shutup
+
+        shutup.mute_warnings()
+        os.chdir(str(self.working_dir))
+
+        # this seems to only  benecessary because we're exec'ing code from a string,
+        # a .py file should be able to import modules from the cwd anyway
+        sys.path.append(str(self.working_dir))
+
+        # capture stdout and stderr
+        # trunk-ignore(mypy/assignment)
+        sys.stdout = sys.stderr = RedirectQueue(result_outq)
+
+    def _run_session(
+        self, code_inq: Queue, result_outq: Queue, event_outq: Queue
+    ) -> None:
+        self.child_proc_setup(result_outq)
+
+        global_scope: dict = {}
+        while True:
+            code = code_inq.get()
+            os.chdir(str(self.working_dir))
+            with open(self.agent_file_name, "w") as f:
+                f.write(code)
+
+            event_outq.put(("state:ready",))
+            try:
+                exec(compile(code, self.agent_file_name, "exec"), global_scope)
+            except BaseException as e:
+                tb_str, e_cls_name, exc_info, exc_stack = exception_summary(
+                    e,
+                    self.working_dir,
+                    self.agent_file_name,
+                    self.format_tb_ipython,
+                )
+                result_outq.put(tb_str)
+                if e_cls_name == "KeyboardInterrupt":
+                    e_cls_name = "TimeoutError"
+
+                event_outq.put(("state:finished", e_cls_name, exc_info, exc_stack))
+            else:
+                event_outq.put(("state:finished", None, None, None))
+
+            # remove the file after execution (otherwise it might be included in the data preview)
+            os.remove(self.agent_file_name)
+
+            # put EOF marker to indicate that we're done
+            result_outq.put("<|EOF|>")
+
+    def create_process(self) -> None:
+        # we use three queues to communicate with the child process:
+        # - code_inq: send code to child to execute
+        # - result_outq: receive stdout/stderr from child
+        # - event_outq: receive events from child (e.g. state:ready, state:finished)
+        # trunk-ignore(mypy/var-annotated)
+        self.code_inq, self.result_outq, self.event_outq = Queue(), Queue(), Queue()
+        self.process = Process(
+            target=self._run_session,
+            args=(self.code_inq, self.result_outq, self.event_outq),
+        )
+        self.process.start()
+
+    def cleanup_session(self):
+        if self.process is None:
+            return
+        try:
+            # Reduce grace period from 2 seconds to 0.5
+            self.process.terminate()
+            self.process.join(timeout=0.5)
+
+            if self.process.exitcode is None:
+                logger.warning("Process failed to terminate, killing immediately")
+                self.process.kill()
+                self.process.join(timeout=0.5)
+
+                if self.process.exitcode is None:
+                    logger.error("Process refuses to die, using SIGKILL")
+                    os.kill(self.process.pid, signal.SIGKILL)
+        except Exception as e:
+            logger.error(f"Error during process cleanup: {e}")
+        finally:
+            if self.process is not None:
+                self.process.close()
+                self.process = None
+
+    def run(self, code: str, reset_session=True) -> ExecutionResult:
+        """
+        Execute the provided Python command in a separate process and return its output.
+
+        Parameters:
+            code (str): Python code to execute.
+            reset_session (bool, optional): Whether to reset the interpreter session before executing the code. Defaults to True.
+
+        Returns:
+            ExecutionResult: Object containing the output and metadata of the code execution.
+
+        """
+
+        logger.debug(f"REPL is executing code (reset_session={reset_session})")
+
+        if reset_session:
+            if self.process is not None:
+                # terminate and clean up previous process
+                self.cleanup_session()
+            self.create_process()
+        else:
+            # reset_session needs to be True on first exec
+            assert self.process is not None
+
+        assert self.process.is_alive()
+
+        self.code_inq.put(code)
+
+        # wait for child to actually start execution (we don't want interrupt child setup)
+        try:
+            state = self.event_outq.get(timeout=10)
+        except queue.Empty:
+            msg = "REPL child process failed to start execution"
+            logger.critical(msg)
+            while not self.result_outq.empty():
+                logger.error(f"REPL output queue dump: {self.result_outq.get()}")
+            raise RuntimeError(msg) from None
+        assert state[0] == "state:ready", state
+        start_time = time.time()
+
+        # this flag indicates that the child ahs exceeded the time limit and an interrupt was sent
+        # if the child process dies without this flag being set, it's an unexpected termination
+        child_in_overtime = False
+
+        while True:
+            try:
+                # check if the child is done
+                state = self.event_outq.get(timeout=1)  # wait for state:finished
+                assert state[0] == "state:finished", state
+                exec_time = time.time() - start_time
+                break
+            except queue.Empty:
+                # we haven't heard back from the child -> check if it's still alive (assuming overtime interrupt wasn't sent yet)
+                if not child_in_overtime and not self.process.is_alive():
+                    msg = "REPL child process died unexpectedly"
+                    logger.critical(msg)
+                    while not self.result_outq.empty():
+                        logger.error(
+                            f"REPL output queue dump: {self.result_outq.get()}"
+                        )
+                    raise RuntimeError(msg) from None
+
+                # child is alive and still executing -> check if we should sigint..
+                if self.timeout is None:
+                    continue
+                running_time = time.time() - start_time
+                if running_time > self.timeout:
+                    logger.warning(f"Execution exceeded timeout of {self.timeout}s")
+                    os.kill(self.process.pid, signal.SIGINT)
+                    child_in_overtime = True
+
+                    # terminate if we're overtime by more than 5 seconds
+                    if running_time > self.timeout + 5:
+                        logger.warning("Child failed to terminate, killing it..")
+                        self.cleanup_session()
+
+                        state = (None, "TimeoutError", {}, [])
+                        exec_time = self.timeout
+                        break
+
+        output: list[str] = []
+        # read all stdout/stderr from child up to the EOF marker
+        # waiting until the queue is empty is not enough since
+        # the feeder thread in child might still be adding to the queue
+        start_collect = time.time()
+        while not self.result_outq.empty() or not output or output[-1] != "<|EOF|>":
+            try:
+                # Add 5-second timeout for output collection
+                if time.time() - start_collect > 5:
+                    logger.warning("Output collection timed out")
+                    break
+                output.append(self.result_outq.get(timeout=1))
+            except queue.Empty:
+                continue
+        output.pop()  # remove the EOF marker
+
+        e_cls_name, exc_info, exc_stack = state[1:]
+
+        if e_cls_name == "TimeoutError":
+            output.append(
+                f"TimeoutError: Execution exceeded the time limit of {humanize.naturaldelta(self.timeout)}"
+            )
+        else:
+            output.append(
+                f"Execution time: {humanize.naturaldelta(exec_time)} seconds (time limit is {humanize.naturaldelta(self.timeout)})."
+            )
+        return ExecutionResult(output, exec_time, e_cls_name, exc_info, exc_stack)

aide/journal.py

@@ -0,0 +1,192 @@
+"""
+The journal is the core datastructure in AIDE that contains:
+- the generated code samples
+- information how code samples relate to each other (the tree structure)
+- code execution results
+- evaluation information such as metrics
+...
+"""
+
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+from dataclasses_json import DataClassJsonMixin
+from .interpreter import ExecutionResult
+from .utils.metric import MetricValue
+from .utils.response import trim_long_string
+
+
+@dataclass(eq=False)
+class Node(DataClassJsonMixin):
+    """A single node in the solution tree. Contains code, execution results, and evaluation information."""
+
+    # ---- code & plan ----
+    code: str
+    plan: str = field(default=None, kw_only=True)  # type: ignore
+
+    # ---- general attrs ----
+    step: int = field(default=None, kw_only=True)  # type: ignore
+    id: str = field(default_factory=lambda: uuid.uuid4().hex, kw_only=True)
+    ctime: float = field(default_factory=lambda: time.time(), kw_only=True)
+    parent: Optional["Node"] = field(default=None, kw_only=True)
+    children: set["Node"] = field(default_factory=set, kw_only=True)
+
+    # ---- execution info ----
+    _term_out: list[str] = field(default=None, kw_only=True)  # type: ignore
+    exec_time: float = field(default=None, kw_only=True)  # type: ignore
+    exc_type: str | None = field(default=None, kw_only=True)
+    exc_info: dict | None = field(default=None, kw_only=True)
+    exc_stack: list[tuple] | None = field(default=None, kw_only=True)
+
+    # ---- evaluation ----
+    # post-execution result analysis (findings/feedback)
+    analysis: str = field(default=None, kw_only=True)  # type: ignore
+    metric: MetricValue = field(default=None, kw_only=True)  # type: ignore
+    # whether the agent decided that the code is buggy
+    # -> always True if exc_type is not None or no valid metric
+    is_buggy: bool = field(default=None, kw_only=True)  # type: ignore
+
+    def __post_init__(self) -> None:
+        if self.parent is not None:
+            self.parent.children.add(self)
+
+    @property
+    def stage_name(self) -> Literal["draft", "debug", "improve"]:
+        """
+        Return the stage of the node:
+        - "stage" if the node is an initial solution draft
+        - "debug" if the node is the result of a debugging step
+        - "improve" if the node is the result of an improvement step
+        """
+        if self.parent is None:
+            return "draft"
+        return "debug" if self.parent.is_buggy else "improve"
+
+    def absorb_exec_result(self, exec_result: ExecutionResult):
+        """Absorb the result of executing the code from this node."""
+        self._term_out = exec_result.term_out
+        self.exec_time = exec_result.exec_time
+        self.exc_type = exec_result.exc_type
+        self.exc_info = exec_result.exc_info
+        self.exc_stack = exec_result.exc_stack
+
+    @property
+    def term_out(self) -> str:
+        """Get the terminal output of the code execution (after truncating it)."""
+        return trim_long_string("".join(self._term_out))
+
+    @property
+    def is_leaf(self) -> bool:
+        """Check if the node is a leaf node in the solution tree."""
+        return not self.children
+
+    def __eq__(self, other):
+        return isinstance(other, Node) and self.id == other.id
+
+    def __hash__(self):
+        return hash(self.id)
+
+    @property
+    def debug_depth(self) -> int:
+        """
+        Length of the current debug path
+        - 0 if the node is not a debug node (parent is not buggy)
+        - 1 if the parent is buggy but the skip parent isn't
+        - n if there were n consecutive debugging steps
+        """
+        if self.stage_name != "debug":
+            return 0
+        return self.parent.debug_depth + 1  # type: ignore
+
+
+@dataclass
+class InteractiveSession(DataClassJsonMixin):
+    """
+    A collection of nodes for an interaction session
+    (when the agent interacts with a Jupyter notebook-like interface).
+    """
+
+    nodes: list[Node] = field(default_factory=list)
+    completed: bool = False
+
+    def append(self, node: Node) -> None:
+        node.step = len(self.nodes)
+        self.nodes.append(node)
+
+    def generate_nb_trace(self, include_prompt, comment_headers=True) -> str:
+        """Generate a trace of the interactive session in IPython format."""
+        trace = []
+        header_prefix = "## " if comment_headers else ""
+        for n in self.nodes:
+            trace.append(f"\n{header_prefix}In [{n.step+1}]:\n")
+            trace.append(n.code)
+            trace.append(f"\n{header_prefix}Out [{n.step+1}]:\n")
+            trace.append(n.term_out)
+
+        if include_prompt and self.nodes:
+            trace.append(f"\n{header_prefix}In [{self.nodes[-1].step+2}]:\n")
+
+        return "\n".join(trace).strip()
+
+
+@dataclass
+class Journal(DataClassJsonMixin):
+    """A collection of nodes representing the solution tree."""
+
+    nodes: list[Node] = field(default_factory=list)
+    # eda: InteractiveSession = field(default_factory=lambda: InteractiveSession())
+
+    def __getitem__(self, idx: int) -> Node:
+        return self.nodes[idx]
+
+    def __len__(self) -> int:
+        """Return the number of nodes in the journal."""
+        return len(self.nodes)
+
+    def append(self, node: Node) -> None:
+        """Append a new node to the journal."""
+        node.step = len(self.nodes)
+        self.nodes.append(node)
+
+    @property
+    def draft_nodes(self) -> list[Node]:
+        """Return a list of nodes representing intial coding drafts"""
+        return [n for n in self.nodes if n.parent is None]
+
+    @property
+    def buggy_nodes(self) -> list[Node]:
+        """Return a list of nodes that are considered buggy by the agent."""
+        return [n for n in self.nodes if n.is_buggy]
+
+    @property
+    def good_nodes(self) -> list[Node]:
+        """Return a list of nodes that are not considered buggy by the agent."""
+        return [n for n in self.nodes if not n.is_buggy]
+
+    def get_metric_history(self) -> list[MetricValue]:
+        """Return a list of all metric values in the journal."""
+        return [n.metric for n in self.nodes]
+
+    def get_best_node(self, only_good=True) -> None | Node:
+        """Return the best solution found so far (node with the highest validation metric)."""
+        if only_good:
+            nodes = self.good_nodes
+            if not nodes:
+                return None
+        else:
+            nodes = self.nodes
+        return max(nodes, key=lambda n: n.metric)
+
+    def generate_summary(self, include_code: bool = False) -> str:
+        """Generate a summary of the journal for the agent."""
+        summary = []
+        for n in self.good_nodes:
+            summary_part = f"Design: {n.plan}\n"
+            if include_code:
+                summary_part += f"Code: {n.code}\n"
+            summary_part += f"Results: {n.analysis}\n"
+            summary_part += f"Validation Metric: {n.metric.value}\n"
+            summary.append(summary_part)
+        return "\n-------------------------------\n".join(summary)

aide/journal2report.py

@@ -0,0 +1,31 @@
+from .backend import query
+from .journal import Journal
+from .utils.config import StageConfig
+
+
+def journal2report(journal: Journal, task_desc: dict, rcfg: StageConfig):
+    """
+    Generate a report from a journal, the report will be in markdown format.
+    """
+    report_input = journal.generate_summary(include_code=True)
+    system_prompt_dict = {
+        "Role": "You are a research assistant that always uses concise language.",
+        "Goal": "The goal is to write a technical report summarising the empirical findings and technical decisions.",
+        "Input": "You are given a raw research journal with list of design attempts and their outcomes, and a task description.",
+        "Output": [
+            "Your output should be a single markdown document.",
+            "Your report should have the following sections: Introduction, Preprocessing, Modellind Methods, Results Discussion, Future Work",
+            "You can include subsections if needed.",
+        ],
+    }
+    context_prompt = (
+        f"Here is the research journal of the agent: <journal>{report_input}<\\journal>, "
+        f"and the task description is: <task>{task_desc}<\\task>."
+    )
+    return query(
+        system_message=system_prompt_dict,
+        user_message=context_prompt,
+        model=rcfg.model,
+        temperature=rcfg.temp,
+        max_tokens=4096,
+    )

aide/run.py

@@ -0,0 +1,148 @@
+import atexit
+import logging
+import shutil
+
+from . import backend
+
+from .agent import Agent
+from .interpreter import Interpreter
+from .journal import Journal, Node
+from .journal2report import journal2report
+from omegaconf import OmegaConf
+from rich.columns import Columns
+from rich.console import Group
+from rich.live import Live
+from rich.padding import Padding
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    TextColumn,
+    TimeRemainingColumn,
+)
+from rich.text import Text
+from rich.status import Status
+from rich.tree import Tree
+from .utils.config import load_task_desc, prep_agent_workspace, save_run, load_cfg
+
+logger = logging.getLogger("aide")
+
+
+def journal_to_rich_tree(journal: Journal):
+    best_node = journal.get_best_node()
+
+    def append_rec(node: Node, tree):
+        if node.is_buggy:
+            s = "[red]◍ bug"
+        else:
+            style = "bold " if node is best_node else ""
+
+            if node is best_node:
+                s = f"[{style}green]● {node.metric.value:.3f} (best)"
+            else:
+                s = f"[{style}green]● {node.metric.value:.3f}"
+
+        subtree = tree.add(s)
+        for child in node.children:
+            append_rec(child, subtree)
+
+    tree = Tree("[bold blue]Solution tree")
+    for n in journal.draft_nodes:
+        append_rec(n, tree)
+    return tree
+
+
+def run():
+    cfg = load_cfg()
+    logger.info(f'Starting run "{cfg.exp_name}"')
+
+    task_desc = load_task_desc(cfg)
+    task_desc_str = backend.compile_prompt_to_md(task_desc)
+
+    with Status("Preparing agent workspace (copying and extracting files) ..."):
+        prep_agent_workspace(cfg)
+
+    def cleanup():
+        if global_step == 0:
+            shutil.rmtree(cfg.workspace_dir)
+
+    atexit.register(cleanup)
+
+    journal = Journal()
+    agent = Agent(
+        task_desc=task_desc,
+        cfg=cfg,
+        journal=journal,
+    )
+    interpreter = Interpreter(
+        cfg.workspace_dir, **OmegaConf.to_container(cfg.exec)  # type: ignore
+    )
+
+    global_step = len(journal)
+    prog = Progress(
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(bar_width=20),
+        MofNCompleteColumn(),
+        TimeRemainingColumn(),
+    )
+    status = Status("[green]Generating code...")
+    prog.add_task("Progress:", total=cfg.agent.steps, completed=global_step)
+
+    def exec_callback(*args, **kwargs):
+        status.update("[magenta]Executing code...")
+        res = interpreter.run(*args, **kwargs)
+        status.update("[green]Generating code...")
+        return res
+
+    def generate_live():
+        tree = journal_to_rich_tree(journal)
+        prog.update(prog.task_ids[0], completed=global_step)
+
+        file_paths = [
+            f"Result visualization:\n[yellow]▶ {str((cfg.log_dir / 'tree_plot.html'))}",
+            f"Agent workspace directory:\n[yellow]▶ {str(cfg.workspace_dir)}",
+            f"Experiment log directory:\n[yellow]▶ {str(cfg.log_dir)}",
+        ]
+        left = Group(
+            Panel(Text(task_desc_str.strip()), title="Task description"), prog, status
+        )
+        right = tree
+        wide = Group(*file_paths)
+
+        return Panel(
+            Group(
+                Padding(wide, (1, 1, 1, 1)),
+                Columns(
+                    [Padding(left, (1, 2, 1, 1)), Padding(right, (1, 1, 1, 2))],
+                    equal=True,
+                ),
+            ),
+            title=f'[b]AIDE is working on experiment: [bold green]"{cfg.exp_name}[/b]"',
+            subtitle="Press [b]Ctrl+C[/b] to stop the run",
+        )
+
+    with Live(
+        generate_live(),
+        refresh_per_second=16,
+        screen=True,
+    ) as live:
+        while global_step < cfg.agent.steps:
+            agent.step(exec_callback=exec_callback)
+            save_run(cfg, journal)
+            global_step = len(journal)
+            live.update(generate_live())
+    interpreter.cleanup_session()
+
+    if cfg.generate_report:
+        print("Generating final report from journal...")
+        report = journal2report(journal, task_desc, cfg.report)
+        print(report)
+        report_file_path = cfg.log_dir / "report.md"
+        with open(report_file_path, "w") as f:
+            f.write(report)
+        print("Report written to file:", report_file_path)
+
+
+if __name__ == "__main__":
+    run()

aide/utils/__init__.py

@@ -0,0 +1,100 @@
+import logging
+import shutil
+import zipfile
+from pathlib import Path
+
+logger = logging.getLogger("aide")
+
+
+def copytree(src: Path, dst: Path, use_symlinks=True):
+    """
+    Copy contents of `src` to `dst`. Unlike shutil.copytree, the dst dir can exist and will be merged.
+    If src is a file, only that file will be copied. Optionally uses symlinks instead of copying.
+
+    Args:
+        src (Path): source directory
+        dst (Path): destination directory
+    """
+    assert dst.is_dir()
+
+    if src.is_file():
+        dest_f = dst / src.name
+        assert not dest_f.exists(), dest_f
+        if use_symlinks:
+            (dest_f).symlink_to(src)
+        else:
+            shutil.copyfile(src, dest_f)
+        return
+
+    for f in src.iterdir():
+        dest_f = dst / f.name
+        assert not dest_f.exists(), dest_f
+        if use_symlinks:
+            (dest_f).symlink_to(f)
+        elif f.is_dir():
+            shutil.copytree(f, dest_f)
+        else:
+            shutil.copyfile(f, dest_f)
+
+
+def clean_up_dataset(path: Path):
+    for item in path.rglob("__MACOSX"):
+        if item.is_dir():
+            shutil.rmtree(item)
+    for item in path.rglob(".DS_Store"):
+        if item.is_file():
+            item.unlink()
+
+
+def extract_archives(path: Path):
+    """
+    unzips all .zip files within `path` and cleans up task dir
+
+    [TODO] handle nested zips
+    """
+    for zip_f in path.rglob("*.zip"):
+        f_out_dir = zip_f.with_suffix("")
+
+        # special case: the intended output path already exists (maybe data has already been extracted by user)
+        if f_out_dir.exists():
+            logger.debug(
+                f"Skipping {zip_f} as an item with the same name already exists."
+            )
+            # if it's a file, it's probably exactly the same as in the zip -> remove the zip
+            # [TODO] maybe add an extra check to see if zip file content matches the colliding file
+            if f_out_dir.is_file() and f_out_dir.suffix != "":
+                zip_f.unlink()
+            continue
+
+        logger.debug(f"Extracting: {zip_f}")
+        f_out_dir.mkdir(exist_ok=True)
+        with zipfile.ZipFile(zip_f, "r") as zip_ref:
+            zip_ref.extractall(f_out_dir)
+
+        # remove any unwanted files
+        clean_up_dataset(f_out_dir)
+
+        contents = list(f_out_dir.iterdir())
+
+        # special case: the zip contains a single dir/file with the same name as the zip
+        if len(contents) == 1 and contents[0].name == f_out_dir.name:
+            sub_item = contents[0]
+            # if it's a dir, move its contents to the parent and remove it
+            if sub_item.is_dir():
+                logger.debug(f"Special handling (child is dir) enabled for: {zip_f}")
+                for f in sub_item.rglob("*"):
+                    shutil.move(f, f_out_dir)
+                sub_item.rmdir()
+            # if it's a file, rename it to the parent and remove the parent
+            elif sub_item.is_file():
+                logger.debug(f"Special handling (child is file) enabled for: {zip_f}")
+                sub_item_tmp = sub_item.rename(f_out_dir.with_suffix(".__tmp_rename"))
+                f_out_dir.rmdir()
+                sub_item_tmp.rename(f_out_dir)
+
+        zip_f.unlink()
+
+
+def preproc_data(path: Path):
+    extract_archives(path)
+    clean_up_dataset(path)

aide/utils/config.py

@@ -0,0 +1,199 @@
+"""configuration and setup utils"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Hashable, cast
+
+import coolname
+import rich
+from omegaconf import OmegaConf
+from rich.syntax import Syntax
+import shutup
+from rich.logging import RichHandler
+import logging
+
+from . import tree_export
+from . import copytree, preproc_data, serialize
+
+shutup.mute_warnings()
+logging.basicConfig(
+    level="WARNING", format="%(message)s", datefmt="[%X]", handlers=[RichHandler()]
+)
+logger = logging.getLogger("aide")
+logger.setLevel(logging.WARNING)
+
+
+""" these dataclasses are just for type hinting, the actual config is in config.yaml """
+
+
+@dataclass
+class StageConfig:
+    model: str
+    temp: float
+
+
+@dataclass
+class SearchConfig:
+    max_debug_depth: int
+    debug_prob: float
+    num_drafts: int
+
+
+@dataclass
+class AgentConfig:
+    steps: int
+    k_fold_validation: int
+    expose_prediction: bool
+    data_preview: bool
+
+    code: StageConfig
+    feedback: StageConfig
+
+    search: SearchConfig
+
+
+@dataclass
+class ExecConfig:
+    timeout: int
+    agent_file_name: str
+    format_tb_ipython: bool
+
+
+@dataclass
+class Config(Hashable):
+    data_dir: Path
+    desc_file: Path | None
+
+    goal: str | None
+    eval: str | None
+
+    log_dir: Path
+    workspace_dir: Path
+
+    preprocess_data: bool
+    copy_data: bool
+
+    exp_name: str
+
+    exec: ExecConfig
+    generate_report: bool
+    report: StageConfig
+    agent: AgentConfig
+
+
+def _get_next_logindex(dir: Path) -> int:
+    """Get the next available index for a log directory."""
+    max_index = -1
+    for p in dir.iterdir():
+        try:
+            if current_index := int(p.name.split("-")[0]) > max_index:
+                max_index = current_index
+        except ValueError:
+            pass
+    return max_index + 1
+
+
+def _load_cfg(
+    path: Path = Path(__file__).parent / "config.yaml", use_cli_args=True
+) -> Config:
+    cfg = OmegaConf.load(path)
+    if use_cli_args:
+        cfg = OmegaConf.merge(cfg, OmegaConf.from_cli())
+    return cfg
+
+
+def load_cfg(path: Path = Path(__file__).parent / "config.yaml") -> Config:
+    """Load config from .yaml file and CLI args, and set up logging directory."""
+    return prep_cfg(_load_cfg(path))
+
+
+def prep_cfg(cfg: Config):
+    if cfg.data_dir is None:
+        raise ValueError("`data_dir` must be provided.")
+
+    if cfg.desc_file is None and cfg.goal is None:
+        raise ValueError(
+            "You must provide either a description of the task goal (`goal=...`) or a path to a plaintext file containing the description (`desc_file=...`)."
+        )
+
+    if cfg.data_dir.startswith("example_tasks/"):
+        cfg.data_dir = Path(__file__).parent.parent / cfg.data_dir
+    cfg.data_dir = Path(cfg.data_dir).resolve()
+
+    if cfg.desc_file is not None:
+        cfg.desc_file = Path(cfg.desc_file).resolve()
+
+    top_log_dir = Path(cfg.log_dir).resolve()
+    top_log_dir.mkdir(parents=True, exist_ok=True)
+
+    top_workspace_dir = Path(cfg.workspace_dir).resolve()
+    top_workspace_dir.mkdir(parents=True, exist_ok=True)
+
+    # generate experiment name and prefix with consecutive index
+    ind = max(_get_next_logindex(top_log_dir), _get_next_logindex(top_workspace_dir))
+    cfg.exp_name = cfg.exp_name or coolname.generate_slug(3)
+    cfg.exp_name = f"{ind}-{cfg.exp_name}"
+
+    cfg.log_dir = (top_log_dir / cfg.exp_name).resolve()
+    cfg.workspace_dir = (top_workspace_dir / cfg.exp_name).resolve()
+
+    # validate the config
+    cfg_schema: Config = OmegaConf.structured(Config)
+    cfg = OmegaConf.merge(cfg_schema, cfg)
+
+    return cast(Config, cfg)
+
+
+def print_cfg(cfg: Config) -> None:
+    rich.print(Syntax(OmegaConf.to_yaml(cfg), "yaml", theme="paraiso-dark"))
+
+
+def load_task_desc(cfg: Config):
+    """Load task description from markdown file or config str."""
+
+    # either load the task description from a file
+    if cfg.desc_file is not None:
+        if not (cfg.goal is None and cfg.eval is None):
+            logger.warning(
+                "Ignoring goal and eval args because task description file is provided."
+            )
+
+        with open(cfg.desc_file) as f:
+            return f.read()
+
+    # or generate it from the goal and eval args
+    if cfg.goal is None:
+        raise ValueError(
+            "`goal` (and optionally `eval`) must be provided if a task description file is not provided."
+        )
+
+    task_desc = {"Task goal": cfg.goal}
+    if cfg.eval is not None:
+        task_desc["Task evaluation"] = cfg.eval
+
+    return task_desc
+
+
+def prep_agent_workspace(cfg: Config):
+    """Setup the agent's workspace and preprocess data if necessary."""
+    (cfg.workspace_dir / "input").mkdir(parents=True, exist_ok=True)
+    (cfg.workspace_dir / "working").mkdir(parents=True, exist_ok=True)
+
+    copytree(cfg.data_dir, cfg.workspace_dir / "input", use_symlinks=not cfg.copy_data)
+    if cfg.preprocess_data:
+        preproc_data(cfg.workspace_dir / "input")
+
+
+def save_run(cfg: Config, journal):
+    cfg.log_dir.mkdir(parents=True, exist_ok=True)
+
+    # save journal
+    serialize.dump_json(journal, cfg.log_dir / "journal.json")
+    # save config
+    OmegaConf.save(config=cfg, f=cfg.log_dir / "config.yaml")
+    # create the tree + code visualization
+    tree_export.generate(cfg, journal, cfg.log_dir / "tree_plot.html")
+    # save the best found solution
+    best_node = journal.get_best_node(only_good=False)
+    with open(cfg.log_dir / "best_solution.py", "w") as f:
+        f.write(best_node.code)

aide/utils/config.yaml

@@ -0,0 +1,58 @@
+# path to the task data directory
+data_dir: null
+
+# either provide a path to a plaintext file describing the task
+desc_file: null
+# or provide the task goal (and optionally evaluation information) as arguments
+goal: null
+eval: null
+
+log_dir: logs
+workspace_dir: workspaces
+
+# whether to unzip any archives in the data directory
+preprocess_data: True
+# whether to copy the data to the workspace directory (otherwise it will be symlinked)
+# copying is recommended to prevent the agent from accidentally modifying the original data
+copy_data: True
+
+exp_name: null # a random experiment name will be generated if not provided
+
+# settings for code execution
+exec:
+  timeout: 3600
+  agent_file_name: runfile.py
+  format_tb_ipython: False
+
+generate_report: True
+# LLM settings for final report from journal
+report:
+  model: gpt-4-turbo
+  temp: 1.0
+
+# agent hyperparams
+agent:
+  # how many improvement iterations to run
+  steps: 20
+  # whether to instruct the agent to use CV (set to 1 to disable)
+  k_fold_validation: 5
+  # whether to instruct the agent to generate a prediction function
+  expose_prediction: False
+  # whether to provide the agent with a preview of the data
+  data_preview: True
+
+  # LLM settings for coding
+  code:
+    model: gpt-4-turbo
+    temp: 0.5
+
+  # LLM settings for evaluating program output / tracebacks
+  feedback:
+    model: gpt-4-turbo
+    temp: 0.5
+
+  # hyperparameters for the tree search
+  search:
+    max_debug_depth: 3
+    debug_prob: 0.5
+    num_drafts: 5

aide/utils/data_preview.py

@@ -0,0 +1,153 @@
+"""
+Contains functions to manually generate a textual preview of some common file types (.csv, .json,..) for the agent.
+"""
+
+import json
+from pathlib import Path
+
+import humanize
+import pandas as pd
+from genson import SchemaBuilder
+from pandas.api.types import is_numeric_dtype
+
+# these files are treated as code (e.g. markdown wrapped)
+code_files = {".py", ".sh", ".yaml", ".yml", ".md", ".html", ".xml", ".log", ".rst"}
+# we treat these files as text (rather than binary) files
+plaintext_files = {".txt", ".csv", ".json", ".tsv"} | code_files
+
+
+def get_file_len_size(f: Path) -> tuple[int, str]:
+    """
+    Calculate the size of a file (#lines for plaintext files, otherwise #bytes)
+    Also returns a human-readable string representation of the size.
+    """
+    if f.suffix in plaintext_files:
+        num_lines = sum(1 for _ in open(f))
+        return num_lines, f"{num_lines} lines"
+    else:
+        s = f.stat().st_size
+        return s, humanize.naturalsize(s)
+
+
+def file_tree(path: Path, depth=0) -> str:
+    """Generate a tree structure of files in a directory"""
+    result = []
+    files = [p for p in Path(path).iterdir() if not p.is_dir()]
+    dirs = [p for p in Path(path).iterdir() if p.is_dir()]
+    max_n = 4 if len(files) > 30 else 8
+    for p in sorted(files)[:max_n]:
+        result.append(f"{' '*depth*4}{p.name} ({get_file_len_size(p)[1]})")
+    if len(files) > max_n:
+        result.append(f"{' '*depth*4}... and {len(files)-max_n} other files")
+
+    for p in sorted(dirs):
+        result.append(f"{' '*depth*4}{p.name}/")
+        result.append(file_tree(p, depth + 1))
+
+    return "\n".join(result)
+
+
+def _walk(path: Path):
+    """Recursively walk a directory (analogous to os.walk but for pathlib.Path)"""
+    for p in sorted(Path(path).iterdir()):
+        if p.is_dir():
+            yield from _walk(p)
+            continue
+        yield p
+
+
+def preview_csv(p: Path, file_name: str, simple=True) -> str:
+    """Generate a textual preview of a csv file
+
+    Args:
+        p (Path): the path to the csv file
+        file_name (str): the file name to use in the preview
+        simple (bool, optional): whether to use a simplified version of the preview. Defaults to True.
+
+    Returns:
+        str: the textual preview
+    """
+    df = pd.read_csv(p)
+
+    out = []
+
+    out.append(f"-> {file_name} has {df.shape[0]} rows and {df.shape[1]} columns.")
+
+    if simple:
+        cols = df.columns.tolist()
+        sel_cols = 15
+        cols_str = ", ".join(cols[:sel_cols])
+        res = f"The columns are: {cols_str}"
+        if len(cols) > sel_cols:
+            res += f"... and {len(cols)-sel_cols} more columns"
+        out.append(res)
+    else:
+        out.append("Here is some information about the columns:")
+        for col in sorted(df.columns):
+            dtype = df[col].dtype
+            name = f"{col} ({dtype})"
+
+            nan_count = df[col].isnull().sum()
+
+            if dtype == "bool":
+                v = df[col][df[col].notnull()].mean()
+                out.append(f"{name} is {v*100:.2f}% True, {100-v*100:.2f}% False")
+            elif df[col].nunique() < 10:
+                out.append(
+                    f"{name} has {df[col].nunique()} unique values: {df[col].unique().tolist()}"
+                )
+            elif is_numeric_dtype(df[col]):
+                out.append(
+                    f"{name} has range: {df[col].min():.2f} - {df[col].max():.2f}, {nan_count} nan values"
+                )
+            elif dtype == "object":
+                out.append(
+                    f"{name} has {df[col].nunique()} unique values. Some example values: {df[col].value_counts().head(4).index.tolist()}"
+                )
+
+    return "\n".join(out)
+
+
+def preview_json(p: Path, file_name: str):
+    """Generate a textual preview of a json file using a generated json schema"""
+    builder = SchemaBuilder()
+    with open(p) as f:
+        builder.add_object(json.load(f))
+    return f"-> {file_name} has auto-generated json schema:\n" + builder.to_json(
+        indent=2
+    )
+
+
+def generate(base_path, include_file_details=True, simple=False):
+    """
+    Generate a textual preview of a directory, including an overview of the directory
+    structure and previews of individual files
+    """
+    tree = f"```\n{file_tree(base_path)}```"
+    out = [tree]
+
+    if include_file_details:
+        for fn in _walk(base_path):
+            file_name = str(fn.relative_to(base_path))
+
+            if fn.suffix == ".csv":
+                out.append(preview_csv(fn, file_name, simple=simple))
+            elif fn.suffix == ".json":
+                out.append(preview_json(fn, file_name))
+            elif fn.suffix in plaintext_files:
+                if get_file_len_size(fn)[0] < 30:
+                    with open(fn) as f:
+                        content = f.read()
+                        if fn.suffix in code_files:
+                            content = f"```\n{content}\n```"
+                        out.append(f"-> {file_name} has content:\n\n{content}")
+
+    result = "\n\n".join(out)
+
+    # if the result is very long we generate a simpler version
+    if len(result) > 6_000 and not simple:
+        return generate(
+            base_path, include_file_details=include_file_details, simple=True
+        )
+
+    return result

aide/utils/metric.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass, field
+from functools import total_ordering
+from typing import Any
+
+import numpy as np
+from dataclasses_json import DataClassJsonMixin
+
+
+@dataclass
+@total_ordering
+class MetricValue(DataClassJsonMixin):
+    """
+    Represents the value of a metric to be optimized, which can be compared to other metric values.
+    Comparisons (and max, min) are based on which value is better, not which is larger.
+    """
+
+    value: float | int | np.number | np.floating | np.ndarray | None
+    maximize: bool | None = field(default=None, kw_only=True)
+
+    def __post_init__(self):
+        if self.value is not None:
+            assert isinstance(self.value, (float, int, np.number, np.floating))
+            self.value = float(self.value)
+
+    def __gt__(self, other) -> bool:
+        """True if self is a _better_ (not necessarily larger) metric value than other"""
+        if self.value is None:
+            return False
+        if other.value is None:
+            return True
+
+        assert type(self) is type(other) and (self.maximize == other.maximize)
+
+        if self.value == other.value:
+            return False
+
+        comp = self.value > other.value
+        return comp if self.maximize else not comp  # type: ignore
+
+    def __eq__(self, other: Any) -> bool:
+        return self.value == other.value
+
+    def __repr__(self) -> str:
+        return str(self)
+
+    def __str__(self) -> str:
+        if self.maximize is None:
+            opt_dir = "?"
+        elif self.maximize:
+            opt_dir = "↑"
+        else:
+            opt_dir = "↓"
+        return f"Metric{opt_dir}({self.value_npsafe:.4f})"
+
+    @property
+    def is_worst(self):
+        """True if the metric value is the worst possible value."""
+        return self.value is None
+
+    @property
+    def value_npsafe(self):
+        return self.value if self.value is not None else float("nan")
+
+
+@dataclass
+class WorstMetricValue(MetricValue):
+    """
+    Represents an invalid metric value, e.g. when the agent creates a buggy solution.
+    Always compares worse than any valid metric value.
+    """
+
+    value: None = None
+
+    def __repr__(self):
+        return super().__repr__()
+
+    def __str__(self):
+        return super().__str__()

aide/utils/response.py

@@ -0,0 +1,91 @@
+import json
+import re
+
+import black
+
+
+def wrap_code(code: str, lang="python") -> str:
+    """Wraps code with three backticks."""
+    return f"```{lang}\n{code}\n```"
+
+
+def is_valid_python_script(script):
+    """Check if a script is a valid Python script."""
+    try:
+        compile(script, "<string>", "exec")
+        return True
+    except SyntaxError:
+        return False
+
+
+def extract_jsons(text):
+    """Extract all JSON objects from the text. Caveat: This function cannot handle nested JSON objects."""
+    json_objects = []
+    matches = re.findall(r"\{.*?\}", text, re.DOTALL)
+    for match in matches:
+        try:
+            json_obj = json.loads(match)
+            json_objects.append(json_obj)
+        except json.JSONDecodeError:
+            pass
+
+    # Sometimes chatgpt-turbo forget the last curly bracket, so we try to add it back when no json is found
+    if len(json_objects) == 0 and not text.endswith("}"):
+        json_objects = extract_jsons(text + "}")
+        if len(json_objects) > 0:
+            return json_objects
+
+    return json_objects
+
+
+def trim_long_string(string, threshold=5100, k=2500):
+    # Check if the length of the string is longer than the threshold
+    if len(string) > threshold:
+        # Output the first k and last k characters
+        first_k_chars = string[:k]
+        last_k_chars = string[-k:]
+
+        truncated_len = len(string) - 2 * k
+
+        return f"{first_k_chars}\n ... [{truncated_len} characters truncated] ... \n{last_k_chars}"
+    else:
+        return string
+
+
+def extract_code(text):
+    """Extract python code blocks from the text."""
+    parsed_codes = []
+
+    # When code is in a text or python block
+    matches = re.findall(r"```(python)?\n*(.*?)\n*```", text, re.DOTALL)
+    for match in matches:
+        code_block = match[1]
+        parsed_codes.append(code_block)
+
+    # When the entire text is code or backticks of the code block is missing
+    if len(parsed_codes) == 0:
+        matches = re.findall(r"^(```(python)?)?\n?(.*?)\n?(```)?$", text, re.DOTALL)
+        if matches:
+            code_block = matches[0][2]
+            parsed_codes.append(code_block)
+
+    # validate the parsed codes
+    valid_code_blocks = [
+        format_code(c) for c in parsed_codes if is_valid_python_script(c)
+    ]
+    return format_code("\n\n".join(valid_code_blocks))
+
+
+def extract_text_up_to_code(s):
+    """Extract (presumed) natural language text up to the start of the first code block."""
+    if "```" not in s:
+        return ""
+    return s[: s.find("```")].strip()
+
+
+def format_code(code) -> str:
+    """Format Python code using Black."""
+    try:
+        return black.format_str(code, mode=black.FileMode())
+    except black.parsing.InvalidInput:  # type: ignore
+        return code

aide/utils/serialize.py

@@ -0,0 +1,51 @@
+import copy
+import json
+from pathlib import Path
+from typing import Type, TypeVar
+
+import dataclasses_json
+from ..journal import Journal
+
+
+def dumps_json(obj: dataclasses_json.DataClassJsonMixin):
+    """Serialize AIDE dataclasses (such as Journals) to JSON."""
+    if isinstance(obj, Journal):
+        obj = copy.deepcopy(obj)
+        node2parent = {n.id: n.parent.id for n in obj.nodes if n.parent is not None}
+        for n in obj.nodes:
+            n.parent = None
+            n.children = set()
+
+    obj_dict = obj.to_dict()
+
+    if isinstance(obj, Journal):
+        obj_dict["node2parent"] = node2parent  # type: ignore
+        obj_dict["__version"] = "2"
+
+    return json.dumps(obj_dict, separators=(",", ":"))
+
+
+def dump_json(obj: dataclasses_json.DataClassJsonMixin, path: Path):
+    with open(path, "w") as f:
+        f.write(dumps_json(obj))
+
+
+G = TypeVar("G", bound=dataclasses_json.DataClassJsonMixin)
+
+
+def loads_json(s: str, cls: Type[G]) -> G:
+    """Deserialize JSON to AIDE dataclasses."""
+    obj_dict = json.loads(s)
+    obj = cls.from_dict(obj_dict)
+
+    if isinstance(obj, Journal):
+        id2nodes = {n.id: n for n in obj.nodes}
+        for child_id, parent_id in obj_dict["node2parent"].items():
+            id2nodes[child_id].parent = id2nodes[parent_id]
+            id2nodes[child_id].__post_init__()
+    return obj
+
+
+def load_json(path: Path, cls: Type[G]) -> G:
+    with open(path, "r") as f:
+        return loads_json(f.read(), cls)

aide/utils/tree_export.py

@@ -0,0 +1,94 @@
+"""Export journal to HTML visualization of tree + code."""
+
+import json
+import textwrap
+from pathlib import Path
+
+import numpy as np
+from igraph import Graph
+from ..journal import Journal
+
+
+def get_edges(journal: Journal):
+    for node in journal:
+        for c in node.children:
+            yield (node.step, c.step)
+
+
+def generate_layout(n_nodes, edges, layout_type="rt"):
+    """Generate visual layout of graph"""
+    layout = Graph(
+        n_nodes,
+        edges=edges,
+        directed=True,
+    ).layout(layout_type)
+    y_max = max(layout[k][1] for k in range(n_nodes))
+    layout_coords = []
+    for n in range(n_nodes):
+        layout_coords.append((layout[n][0], 2 * y_max - layout[n][1]))
+    return np.array(layout_coords)
+
+
+def normalize_layout(layout: np.ndarray):
+    """Normalize layout to [0, 1]"""
+    layout = (layout - layout.min(axis=0)) / (layout.max(axis=0) - layout.min(axis=0))
+    layout[:, 1] = 1 - layout[:, 1]
+    layout[:, 1] = np.nan_to_num(layout[:, 1], nan=0)
+    layout[:, 0] = np.nan_to_num(layout[:, 0], nan=0.5)
+    return layout
+
+
+def strip_code_markers(code: str) -> str:
+    """Remove markdown code block markers if present."""
+    code = code.strip()
+    if code.startswith("```"):
+        # Remove opening backticks and optional language identifier
+        first_newline = code.find("\n")
+        if first_newline != -1:
+            code = code[first_newline:].strip()
+    if code.endswith("```"):
+        code = code[:-3].strip()
+    return code
+
+
+def cfg_to_tree_struct(cfg, jou: Journal):
+    edges = list(get_edges(jou))
+    layout = normalize_layout(generate_layout(len(jou), edges))
+
+    # metrics = np.array([n.metric.value_npsafe for n in jou])
+    # metrics = (metrics - np.nanmin(metrics)) / (np.nanmax(metrics) - np.nanmin(metrics))
+    # metrics = np.nan_to_num(metrics, nan=1)
+    # metrics[:] = 0
+    metrics = np.array([0 for n in jou])
+
+    return dict(
+        edges=edges,
+        layout=layout.tolist(),
+        plan=[textwrap.fill(n.plan, width=80) for n in jou.nodes],
+        code=[strip_code_markers(n.code) for n in jou],
+        term_out=[n.term_out for n in jou],
+        analysis=[n.analysis for n in jou],
+        exp_name=cfg.exp_name,
+        metrics=metrics.tolist(),
+    )
+
+
+def generate_html(tree_graph_str: str):
+    template_dir = Path(__file__).parent / "viz_templates"
+
+    with open(template_dir / "template.js") as f:
+        js = f.read()
+        js = js.replace("<placeholder>", tree_graph_str)
+
+    with open(template_dir / "template.html") as f:
+        html = f.read()
+        html = html.replace("<!-- placeholder -->", js)
+
+        return html
+
+
+def generate(cfg, jou: Journal, out_path: Path):
+    tree_graph_str = json.dumps(cfg_to_tree_struct(cfg, jou))
+    html = generate_html(tree_graph_str)
+    with open(out_path, "w") as f:
+        f.write(html)

aide/utils/viz_templates/template.html

@@ -0,0 +1,61 @@
+<!doctype html>
+<html lang="en"> 
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <script
+      id="p5scripttag"
+      src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/1.9.0/p5.min.js"
+      integrity="sha512-uaz5GpnQoE6t5echKlX8P52czvsIGgLPcvlzfvRubLZ1Hp8JemUDnbUiAahbVtPb+jUVrNETuXvAhDDF/N3M4w=="
+      crossorigin="anonymous"
+      referrerpolicy="no-referrer"
+    ></script>
+
+    <link
+      rel="stylesheet"
+      href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/atom-one-dark.min.css"
+    />
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/languages/python.min.js"></script>
+
+    <script>
+      <!-- placeholder -->
+    </script>
+    <title>AIDE Run Visualization</title>
+    <style>
+      body,
+      * {
+        margin: 0;
+        padding: 0;
+        box-sizing: border-box;
+        overflow: scroll;
+      }
+      body {
+        background-color: #f2f0e7;
+      }
+      canvas {
+        float: left;
+        height: 100vh;
+        width: 40vw;
+      }
+      #text-container {
+        float: right;
+        height: 100vh;
+        width: 50vw;
+        background-color: #282c34;
+      }
+      #plan {
+        /* border-left: 2px solid #282c34; */
+        background-color: #282c34;
+        color: #f2f0e7;
+        min-height: 5rem;
+        padding: 1em 0 1em 1em;
+      }
+    </style>
+  </head>
+  <body>
+    <pre
+      id="text-container"
+    ><div id="plan"></div><hr><code id="code" class="language-python"></code></pre>
+  </body>
+</html>

aide/utils/viz_templates/template.js

@@ -0,0 +1,362 @@
+const bgCol = "#F2F0E7";
+const accentCol = "#fd4578";
+
+hljs.initHighlightingOnLoad();
+
+const updateTargetDims = () => {
+  // width is max-width of `.contentContainer` - its padding
+  // return [min(windowWidth, 900 - 80), 700]
+  return [windowWidth * (1 / 2), windowHeight];
+};
+
+const setCodeAndPlan = (code, plan) => {
+  const codeElm = document.getElementById("code");
+  if (codeElm) {
+    // codeElm.innerText = code;
+    codeElm.innerHTML = hljs.highlight(code, { language: "python" }).value;
+  }
+
+  const planElm = document.getElementById("plan");
+  if (planElm) {
+    // planElm.innerText = plan.trim();
+    planElm.innerHTML = hljs.highlight(plan, { language: "plaintext" }).value;
+  }
+};
+
+windowResized = () => {
+  resizeCanvas(...updateTargetDims());
+  awaitingPostResizeOps = true;
+};
+
+const animEase = (t) => 1 - (1 - Math.min(t, 1.0)) ** 5;
+
+// ---- global constants ----
+
+const globalAnimSpeed = 1.1;
+const scaleFactor = 0.57;
+
+// ---- global vars ----
+
+let globalTime = 0;
+let manualSelection = false;
+
+let currentElemInd = 0;
+
+let treeStructData = <placeholder>
+
+let lastClick = 0;
+let firstFrameTime = undefined;
+
+let nodes = [];
+let edges = [];
+
+let lastScrollPos = 0;
+
+setup = () => {
+  canvas = createCanvas(...updateTargetDims());
+};
+
+class Node {
+  x;
+  y;
+  size;
+  xT;
+  yT;
+  xB;
+  yB;
+  treeInd;
+  color;
+  relSize;
+  animationStart = Number.MAX_VALUE;
+  animationProgress = 0;
+  isStatic = false;
+  hasChildren = false;
+  isRootNode = true;
+  isStarred = false;
+  selected = false;
+  renderSize = 10;
+  edges = [];
+  bgCol;
+
+  constructor(x, y, relSize, treeInd) {
+    const minSize = 35;
+    const maxSize = 60;
+
+    const maxColor = 10;
+    const minColor = 125;
+
+    this.relSize = relSize;
+    this.treeInd = treeInd;
+    this.size = minSize + (maxSize - minSize) * relSize;
+    this.color = minColor + (maxColor - minColor) * relSize;
+    this.bgCol = Math.round(Math.max(this.color / 2, 0));
+
+    this.x = x;
+    this.y = y;
+    this.xT = x;
+    this.yT = y - this.size / 2;
+    this.xB = x;
+    this.yB = y + this.size / 2;
+
+    nodes.push(this);
+  }
+
+  startAnimation = (offset = 0) => {
+    if (this.animationStart == Number.MAX_VALUE)
+      this.animationStart = globalTime + offset;
+  };
+
+  child = (node) => {
+    let edge = new Edge(this, node);
+    this.edges.push(edge);
+    edges.push(edge);
+    this.hasChildren = true;
+    node.isRootNode = false;
+    return node;
+  };
+
+  render = () => {
+    if (globalTime - this.animationStart < 0) return;
+
+    const mouseXlocalCoords = (mouseX - width / 2) / scaleFactor;
+    const mouseYlocalCoords = (mouseY - height / 2) / scaleFactor;
+    const isMouseOver =
+      dist(mouseXlocalCoords, mouseYlocalCoords, this.x, this.y) <
+      this.renderSize / 1.5;
+    if (isMouseOver) cursor(HAND);
+    if (isMouseOver && mouseIsPressed) {
+      nodes.forEach((n) => (n.selected = false));
+      this.selected = true;
+      setCodeAndPlan(
+        treeStructData.code[this.treeInd],
+        treeStructData.plan[this.treeInd],
+      );
+      manualSelection = true;
+    }
+
+    this.renderSize = this.size;
+    if (!this.isStatic) {
+      this.animationProgress = animEase(
+        (globalTime - this.animationStart) / 1000,
+      );
+      if (this.animationProgress >= 1) {
+        this.isStatic = true;
+      } else {
+        this.renderSize =
+          this.size *
+          (0.8 +
+            0.2 *
+              (-3.33 * this.animationProgress ** 2 +
+                4.33 * this.animationProgress));
+      }
+    }
+
+    fill(this.color);
+    if (this.selected) {
+      fill(accentCol);
+    }
+
+    noStroke();
+    square(
+      this.x - this.renderSize / 2,
+      this.y - this.renderSize / 2,
+      this.renderSize,
+      10,
+    );
+
+    noStroke();
+    textAlign(CENTER, CENTER);
+    textSize(this.renderSize / 2);
+    fill(255);
+    // fill(lerpColor(color(accentCol), color(255), this.animationProgress))
+    text("{ }", this.x, this.y - 1);
+    // DEBUG PRINT:
+    // text(round(this.relSize, 2), this.x, this.y - 1)
+    // text(this.treeInd, this.x, this.y + 15)
+
+    const dotAnimThreshold = 0.85;
+    if (this.isStarred && this.animationProgress >= dotAnimThreshold) {
+      let dotAnimProgress =
+        (this.animationProgress - dotAnimThreshold) / (1 - dotAnimThreshold);
+      textSize(
+        ((-3.33 * dotAnimProgress ** 2 + 4.33 * dotAnimProgress) *
+          this.renderSize) /
+          2,
+      );
+      if (this.selected) {
+        fill(0);
+        stroke(0);
+      } else {
+        fill(accentCol);
+        stroke(accentCol);
+      }
+      strokeWeight((-(dotAnimProgress ** 2) + dotAnimProgress) * 2);
+      text("*", this.x + 20, this.y - 11);
+      noStroke();
+    }
+
+    if (!this.isStatic) {
+      fill(bgCol);
+      const progressAnimBaseSize = this.renderSize + 5;
+      rect(
+        this.x - progressAnimBaseSize / 2,
+        this.y -
+          progressAnimBaseSize / 2 +
+          progressAnimBaseSize * this.animationProgress,
+        progressAnimBaseSize,
+        progressAnimBaseSize * (1 - this.animationProgress),
+      );
+    }
+    if (this.animationProgress >= 0.9) {
+      this.edges
+        .sort((a, b) => a.color() - b.color())
+        .forEach((e, i) => {
+          e.startAnimation((i / this.edges.length) ** 2 * 1000);
+        });
+    }
+  };
+}
+
+class Edge {
+  nodeT;
+  nodeB;
+  animX = 0;
+  animY = 0;
+  animationStart = Number.MAX_VALUE;
+  animationProgress = 0;
+  isStatic = false;
+  weight = 0;
+
+  constructor(nodeT, nodeB) {
+    this.nodeT = nodeT;
+    this.nodeB = nodeB;
+    this.weight = 2 + nodeB.relSize * 1;
+  }
+
+  color = () => this.nodeB.color;
+
+  startAnimation = (offset = 0) => {
+    if (this.animationStart == Number.MAX_VALUE)
+      this.animationStart = globalTime + offset;
+  };
+
+  render = () => {
+    if (globalTime - this.animationStart < 0) return;
+
+    if (!this.isStatic) {
+      this.animationProgress = animEase(
+        (globalTime - this.animationStart) / 1000,
+      );
+      if (this.animationProgress >= 1) {
+        this.isStatic = true;
+        this.animX = this.nodeB.xT;
+        this.animY = this.nodeB.yT;
+      } else {
+        this.animX = bezierPoint(
+          this.nodeT.xB,
+          this.nodeT.xB,
+          this.nodeB.xT,
+          this.nodeB.xT,
+          this.animationProgress,
+        );
+
+        this.animY = bezierPoint(
+          this.nodeT.yB,
+          (this.nodeT.yB + this.nodeB.yT) / 2,
+          (this.nodeT.yB + this.nodeB.yT) / 2,
+          this.nodeB.yT,
+          this.animationProgress,
+        );
+      }
+    }
+    if (this.animationProgress >= 0.97) {
+      this.nodeB.startAnimation();
+    }
+
+    strokeWeight(this.weight);
+    noFill();
+    stroke(
+      lerpColor(color(bgCol), color(accentCol), this.nodeB.relSize * 1 + 0.7),
+    );
+    bezier(
+      this.nodeT.xB,
+      this.nodeT.yB,
+      this.nodeT.xB,
+      (this.nodeT.yB + this.nodeB.yT) / 2,
+      this.animX,
+      (this.nodeT.yB + this.nodeB.yT) / 2,
+      this.animX,
+      this.animY,
+    );
+  };
+}
+
+draw = () => {
+  cursor(ARROW);
+  frameRate(120);
+  if (!firstFrameTime && frameCount <= 1) {
+    firstFrameTime = millis();
+  }
+  // ---- update global animation state ----
+  const initialSpeedScalingEaseIO =
+    (cos(min((millis() - firstFrameTime) / 8000, 1.0) * PI) + 1) / 2;
+  const initialSpeedScalingEase =
+    (cos(min((millis() - firstFrameTime) / 8000, 1.0) ** (1 / 2) * PI) + 1) / 2;
+  const initAnimationSpeedFactor = 1.0 - 0.4 * initialSpeedScalingEaseIO;
+  // update global scaling-aware clock
+  globalTime += globalAnimSpeed * initAnimationSpeedFactor * deltaTime;
+
+  if (nodes.length == 0) {
+    const spacingHeight = height * 1.3;
+    const spacingWidth = width * 1.3;
+    treeStructData.layout.forEach((lay, index) => {
+      new Node(
+        spacingWidth * lay[0] - spacingWidth / 2,
+        20 + spacingHeight * lay[1] - spacingHeight / 2,
+        1 - treeStructData.metrics[index],
+        index,
+      );
+    });
+    treeStructData.edges.forEach((ind) => {
+      nodes[ind[0]].child(nodes[ind[1]]);
+    });
+    nodes.forEach((n) => {
+      if (n.isRootNode) n.startAnimation();
+    });
+    nodes[0].selected = true;
+    setCodeAndPlan(
+      treeStructData.code[0],
+      treeStructData.plan[0],
+    )
+  }
+
+  const staticNodes = nodes.filter(
+    (n) => n.isStatic || n.animationProgress >= 0.7,
+  );
+  if (staticNodes.length > 0) {
+    const largestNode = staticNodes.reduce((prev, current) =>
+      prev.relSize > current.relSize ? prev : current,
+    );
+    if (!manualSelection) {
+      if (!largestNode.selected) {
+        setCodeAndPlan(
+          treeStructData.code[largestNode.treeInd],
+          treeStructData.plan[largestNode.treeInd],
+        );
+      }
+      staticNodes.forEach((node) => {
+        node.selected = node === largestNode;
+      });
+    }
+  }
+  background(bgCol);
+  // global animation transforms
+  translate(width / 2, height / 2);
+  scale(scaleFactor);
+
+  
+  // ---- fg render ----
+  edges.forEach((e) => e.render());
+  nodes.forEach((n) => n.render());
+  
+};

aide/webui/app.py

@@ -0,0 +1,599 @@
+import streamlit as st
+import streamlit.components.v1 as components
+from pathlib import Path
+import tempfile
+import shutil
+import os
+import json
+from omegaconf import OmegaConf
+from rich.console import Console
+import sys
+from dotenv import load_dotenv
+import logging
+from aide import Experiment
+
+# Set up logging configuration
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[logging.StreamHandler(sys.stderr)],
+)
+
+logger = logging.getLogger("aide")
+logger.setLevel(logging.INFO)
+
+console = Console(file=sys.stderr)
+
+
+class WebUI:
+    """
+    WebUI encapsulates the Streamlit application logic for the AIDE Machine Learning Engineer Agent.
+    """
+
+    def __init__(self):
+        """
+        Initialize the WebUI with environment variables and session state.
+        """
+        self.env_vars = self.load_env_variables()
+        self.project_root = Path(__file__).parent.parent.parent
+        self.config_session_state()
+        self.setup_page()
+
+    @staticmethod
+    def load_env_variables():
+        """
+        Load API keys and environment variables from .env file.
+
+        Returns:
+            dict: Dictionary containing API keys.
+        """
+        load_dotenv()
+        return {
+            "openai_key": os.getenv("OPENAI_API_KEY", ""),
+            "anthropic_key": os.getenv("ANTHROPIC_API_KEY", ""),
+        }
+
+    @staticmethod
+    def config_session_state():
+        """
+        Configure default values for Streamlit session state.
+        """
+        if "is_running" not in st.session_state:
+            st.session_state.is_running = False
+        if "current_step" not in st.session_state:
+            st.session_state.current_step = 0
+        if "total_steps" not in st.session_state:
+            st.session_state.total_steps = 0
+        if "progress" not in st.session_state:
+            st.session_state.progress = 0
+        if "results" not in st.session_state:
+            st.session_state.results = None
+
+    @staticmethod
+    def setup_page():
+        """
+        Set up the Streamlit page configuration and load custom CSS.
+        """
+        st.set_page_config(
+            page_title="AIDE: Machine Learning Engineer Agent",
+            layout="wide",
+        )
+        WebUI.load_css()
+
+    @staticmethod
+    def load_css():
+        """
+        Load custom CSS styles from 'style.css' file.
+        """
+        css_file = Path(__file__).parent / "style.css"
+        if css_file.exists():
+            with open(css_file) as f:
+                st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
+        else:
+            st.warning(f"CSS file not found at: {css_file}")
+
+    def run(self):
+        """
+        Run the main logic of the Streamlit application.
+        """
+        self.render_sidebar()
+        input_col, results_col = st.columns([1, 3])
+        with input_col:
+            self.render_input_section(results_col)
+
+    def render_sidebar(self):
+        """
+        Render the sidebar with API key settings.
+        """
+        with st.sidebar:
+            st.header("⚙️ Settings")
+            st.markdown(
+                "<p style='text-align: center;'>OpenAI API Key</p>",
+                unsafe_allow_html=True,
+            )
+            openai_key = st.text_input(
+                "OpenAI API Key",
+                value=self.env_vars["openai_key"],
+                type="password",
+                label_visibility="collapsed",
+            )
+            st.markdown(
+                "<p style='text-align: center;'>Anthropic API Key</p>",
+                unsafe_allow_html=True,
+            )
+            anthropic_key = st.text_input(
+                "Anthropic API Key",
+                value=self.env_vars["anthropic_key"],
+                type="password",
+                label_visibility="collapsed",
+            )
+            if st.button("Save API Keys", use_container_width=True):
+                st.session_state.openai_key = openai_key
+                st.session_state.anthropic_key = anthropic_key
+                st.success("API keys saved!")
+
+    def render_input_section(self, results_col):
+        """
+        Render the input section of the application.
+
+        Args:
+            results_col (st.delta_generator.DeltaGenerator): The results column to pass to methods.
+        """
+        st.header("Input")
+        uploaded_files = self.handle_file_upload()
+        goal_text, eval_text, num_steps = self.handle_user_inputs()
+        if st.button("Run AIDE", type="primary", use_container_width=True):
+            with st.spinner("AIDE is running..."):
+                results = self.run_aide(
+                    uploaded_files, goal_text, eval_text, num_steps, results_col
+                )
+                st.session_state.results = results
+
+    def handle_file_upload(self):
+        """
+        Handle file uploads and example file loading.
+
+        Returns:
+            list: List of uploaded or example files.
+        """
+        # Only show file uploader if no example files are loaded
+        if not st.session_state.get("example_files"):
+            uploaded_files = st.file_uploader(
+                "Upload Data Files",
+                accept_multiple_files=True,
+                type=["csv", "txt", "json", "md"],
+                label_visibility="collapsed",
+            )
+
+            if uploaded_files:
+                st.session_state.pop(
+                    "example_files", None
+                )  # Remove example files if any
+                return uploaded_files
+
+            # Only show example button if no files are uploaded
+            if st.button(
+                "Load Example Experiment", type="primary", use_container_width=True
+            ):
+                st.session_state.example_files = self.load_example_files()
+
+        if st.session_state.get("example_files"):
+            st.info("Example files loaded! Click 'Run AIDE' to proceed.")
+            with st.expander("View Loaded Files", expanded=False):
+                for file in st.session_state.example_files:
+                    st.text(f"📄 {file['name']}")
+            return st.session_state.example_files
+
+        return []  # Return empty list if no files are uploaded or loaded
+
+    def handle_user_inputs(self):
+        """
+        Handle goal, evaluation criteria, and number of steps inputs.
+
+        Returns:
+            tuple: Goal text, evaluation criteria text, and number of steps.
+        """
+        goal_text = st.text_area(
+            "Goal",
+            value=st.session_state.get("goal", ""),
+            placeholder="Example: Predict the sales price for each house",
+        )
+        eval_text = st.text_area(
+            "Evaluation Criteria",
+            value=st.session_state.get("eval", ""),
+            placeholder="Example: Use the RMSE metric between the logarithm of the predicted and observed values.",
+        )
+        num_steps = st.slider(
+            "Number of Steps",
+            min_value=1,
+            max_value=20,
+            value=st.session_state.get("steps", 10),
+        )
+        return goal_text, eval_text, num_steps
+
+    @staticmethod
+    def load_example_files():
+        """
+        Load example files from the 'example_tasks/house_prices' directory.
+
+        Returns:
+            list: List of example files with their paths.
+        """
+        package_root = Path(__file__).parent.parent
+        example_dir = package_root / "example_tasks" / "house_prices"
+
+        if not example_dir.exists():
+            st.error(f"Example directory not found at: {example_dir}")
+            return []
+
+        example_files = []
+
+        for file_path in example_dir.glob("*"):
+            if file_path.suffix.lower() in [".csv", ".txt", ".json", ".md"]:
+                with tempfile.NamedTemporaryFile(
+                    delete=False, suffix=file_path.suffix
+                ) as tmp_file:
+                    tmp_file.write(file_path.read_bytes())
+                    example_files.append(
+                        {"name": file_path.name, "path": tmp_file.name}
+                    )
+
+        if not example_files:
+            st.warning("No example files found in the example directory")
+
+        st.session_state["goal"] = "Predict the sales price for each house"
+        st.session_state["eval"] = (
+            "Use the RMSE metric between the logarithm of the predicted and observed values."
+        )
+
+        return example_files
+
+    def run_aide(self, files, goal_text, eval_text, num_steps, results_col):
+        """
+        Run the AIDE experiment with the provided inputs.
+
+        Args:
+            files (list): List of uploaded or example files.
+            goal_text (str): The goal of the experiment.
+            eval_text (str): The evaluation criteria.
+            num_steps (int): Number of steps to run.
+            results_col (st.delta_generator.DeltaGenerator): Results column for displaying progress.
+
+        Returns:
+            dict: Dictionary containing the results of the experiment.
+        """
+        try:
+            self.initialize_run_state(num_steps)
+            self.set_api_keys()
+
+            input_dir = self.prepare_input_directory(files)
+            if not input_dir:
+                return None
+
+            experiment = self.initialize_experiment(input_dir, goal_text, eval_text)
+
+            # Create separate placeholders for progress and config
+            progress_placeholder = results_col.empty()
+            config_placeholder = results_col.empty()
+            results_placeholder = results_col.empty()
+
+            for step in range(num_steps):
+                st.session_state.current_step = step + 1
+                progress = (step + 1) / num_steps
+
+                # Update progress
+                with progress_placeholder.container():
+                    st.markdown(
+                        f"### 🔥 Running Step {st.session_state.current_step}/{st.session_state.total_steps}"
+                    )
+                    st.progress(progress)
+
+                # Show config only for first step
+                if step == 0:
+                    with config_placeholder.container():
+                        st.markdown("### 📋 Configuration")
+                        st.code(OmegaConf.to_yaml(experiment.cfg), language="yaml")
+
+                experiment.run(steps=1)
+
+                # Show results
+                with results_placeholder.container():
+                    self.render_live_results(experiment)
+
+                # Clear config after first step
+                if step == 0:
+                    config_placeholder.empty()
+
+            # Clear progress after all steps
+            progress_placeholder.empty()
+
+            # Update session state
+            st.session_state.is_running = False
+            st.session_state.results = self.collect_results(experiment)
+            return st.session_state.results
+
+        except Exception as e:
+            st.session_state.is_running = False
+            console.print_exception()
+            st.error(f"Error occurred: {str(e)}")
+            return None
+
+    @staticmethod
+    def initialize_run_state(num_steps):
+        """
+        Initialize the running state for the experiment.
+
+        Args:
+            num_steps (int): Total number of steps in the experiment.
+        """
+        st.session_state.is_running = True
+        st.session_state.current_step = 0
+        st.session_state.total_steps = num_steps
+        st.session_state.progress = 0
+
+    @staticmethod
+    def set_api_keys():
+        """
+        Set the API keys in the environment variables from the session state.
+        """
+        if st.session_state.get("openai_key"):
+            os.environ["OPENAI_API_KEY"] = st.session_state.openai_key
+        if st.session_state.get("anthropic_key"):
+            os.environ["ANTHROPIC_API_KEY"] = st.session_state.anthropic_key
+
+    def prepare_input_directory(self, files):
+        """
+        Prepare the input directory and handle uploaded files.
+
+        Args:
+            files (list): List of uploaded or example files.
+
+        Returns:
+            Path: The input directory path, or None if files are missing.
+        """
+        input_dir = self.project_root / "input"
+        input_dir.mkdir(parents=True, exist_ok=True)
+
+        if files:
+            for file in files:
+                if isinstance(file, dict):  # Example files
+                    shutil.copy2(file["path"], input_dir / file["name"])
+                else:  # Uploaded files
+                    with open(input_dir / file.name, "wb") as f:
+                        f.write(file.getbuffer())
+        else:
+            st.error("Please upload data files")
+            return None
+        return input_dir
+
+    @staticmethod
+    def initialize_experiment(input_dir, goal_text, eval_text):
+        """
+        Initialize the AIDE Experiment.
+
+        Args:
+            input_dir (Path): Path to the input directory.
+            goal_text (str): The goal of the experiment.
+            eval_text (str): The evaluation criteria.
+
+        Returns:
+            Experiment: The initialized Experiment object.
+        """
+        experiment = Experiment(data_dir=str(input_dir), goal=goal_text, eval=eval_text)
+        return experiment
+
+    @staticmethod
+    def collect_results(experiment):
+        """
+        Collect the results from the experiment.
+
+        Args:
+            experiment (Experiment): The Experiment object.
+
+        Returns:
+            dict: Dictionary containing the collected results.
+        """
+        solution_path = experiment.cfg.log_dir / "best_solution.py"
+        if solution_path.exists():
+            solution = solution_path.read_text()
+        else:
+            solution = "No solution found"
+
+        journal_data = [
+            {
+                "step": node.step,
+                "code": str(node.code),
+                "metric": str(node.metric.value) if node.metric else None,
+                "is_buggy": node.is_buggy,
+            }
+            for node in experiment.journal.nodes
+        ]
+
+        results = {
+            "solution": solution,
+            "config": OmegaConf.to_yaml(experiment.cfg),
+            "journal": json.dumps(journal_data, indent=2, default=str),
+            "tree_path": str(experiment.cfg.log_dir / "tree_plot.html"),
+        }
+        return results
+
+    @staticmethod
+    def render_tree_visualization(results):
+        """
+        Render the tree visualization from the experiment results.
+
+        Args:
+            results (dict): The results dictionary containing paths and data.
+        """
+        if "tree_path" in results:
+            tree_path = Path(results["tree_path"])
+            logger.info(f"Loading tree visualization from: {tree_path}")
+            if tree_path.exists():
+                with open(tree_path, "r", encoding="utf-8") as f:
+                    html_content = f.read()
+                components.html(html_content, height=600, scrolling=True)
+            else:
+                st.error(f"Tree visualization file not found at: {tree_path}")
+                logger.error(f"Tree file not found at: {tree_path}")
+        else:
+            st.info("No tree visualization available for this run.")
+
+    @staticmethod
+    def render_best_solution(results):
+        """
+        Display the best solution code.
+
+        Args:
+            results (dict): The results dictionary containing the solution.
+        """
+        if "solution" in results:
+            solution_code = results["solution"]
+            st.code(solution_code, language="python")
+        else:
+            st.info("No solution available.")
+
+    @staticmethod
+    def render_config(results):
+        """
+        Display the configuration used in the experiment.
+
+        Args:
+            results (dict): The results dictionary containing the config.
+        """
+        if "config" in results:
+            st.code(results["config"], language="yaml")
+        else:
+            st.info("No configuration available.")
+
+    @staticmethod
+    def render_journal(results):
+        """
+        Display the experiment journal as JSON.
+
+        Args:
+            results (dict): The results dictionary containing the journal.
+        """
+        if "journal" in results:
+            try:
+                journal_data = json.loads(results["journal"])
+                formatted_journal = json.dumps(journal_data, indent=2)
+                st.code(formatted_journal, language="json")
+            except json.JSONDecodeError:
+                st.code(results["journal"], language="json")
+        else:
+            st.info("No journal available.")
+
+    @staticmethod
+    def get_best_metric(results):
+        """
+        Extract the best validation metric from results.
+        """
+        try:
+            journal_data = json.loads(results["journal"])
+            metrics = []
+            for node in journal_data:
+                if node["metric"] is not None:
+                    try:
+                        # Convert string metric to float
+                        metric_value = float(node["metric"])
+                        metrics.append(metric_value)
+                    except (ValueError, TypeError):
+                        continue
+            return max(metrics) if metrics else None
+        except (json.JSONDecodeError, KeyError):
+            return None
+
+    @staticmethod
+    def render_validation_plot(results, step):
+        """
+        Render the validation score plot.
+
+        Args:
+            results (dict): The results dictionary
+            step (int): Current step number for unique key generation
+        """
+        try:
+            journal_data = json.loads(results["journal"])
+            steps = []
+            metrics = []
+
+            for node in journal_data:
+                if node["metric"] is not None and node["metric"].lower() != "none":
+                    try:
+                        metric_value = float(node["metric"])
+                        steps.append(node["step"])
+                        metrics.append(metric_value)
+                    except (ValueError, TypeError):
+                        continue
+
+            if metrics:
+                import plotly.graph_objects as go
+
+                fig = go.Figure()
+                fig.add_trace(
+                    go.Scatter(
+                        x=steps,
+                        y=metrics,
+                        mode="lines+markers",
+                        name="Validation Score",
+                        line=dict(color="#F04370"),
+                        marker=dict(color="#F04370"),
+                    )
+                )
+
+                fig.update_layout(
+                    title="Validation Score Progress",
+                    xaxis_title="Step",
+                    yaxis_title="Validation Score",
+                    template="plotly_white",
+                    hovermode="x unified",
+                    plot_bgcolor="rgba(0,0,0,0)",
+                    paper_bgcolor="rgba(0,0,0,0)",
+                )
+
+                # Only keep the key for plotly_chart
+                st.plotly_chart(fig, use_container_width=True, key=f"plot_{step}")
+            else:
+                st.info("No validation metrics available to plot")
+
+        except (json.JSONDecodeError, KeyError):
+            st.error("Could not parse validation metrics data")
+
+    def render_live_results(self, experiment):
+        """
+        Render live results.
+
+        Args:
+            experiment (Experiment): The Experiment object
+        """
+        results = self.collect_results(experiment)
+
+        # Create tabs for different result views
+        tabs = st.tabs(
+            [
+                "Tree Visualization",
+                "Best Solution",
+                "Config",
+                "Journal",
+                "Validation Plot",
+            ]
+        )
+
+        with tabs[0]:
+            self.render_tree_visualization(results)
+        with tabs[1]:
+            self.render_best_solution(results)
+        with tabs[2]:
+            self.render_config(results)
+        with tabs[3]:
+            self.render_journal(results)
+        with tabs[4]:
+            best_metric = self.get_best_metric(results)
+            if best_metric is not None:
+                st.metric("Best Validation Score", f"{best_metric:.4f}")
+            self.render_validation_plot(results, step=st.session_state.current_step)
+
+
+if __name__ == "__main__":
+    app = WebUI()
+    app.run()

aide/webui/style.css

@@ -0,0 +1,173 @@
+.stVerticalBlock {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+
+.block-container {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+
+[data-testid="stVerticalBlock"] {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+.st-emotion-cache-40pabj{
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+/* Make the toolbar transparent */
+.st-emotion-cache-18ni7ap {
+    pointer-events: none;
+    background: rgb(255 255 255 / 0%)
+    }
+
+.stAppHeader {
+    display: none;
+}
+
+header {visibility: hidden;}
+#MainMenu {visibility: hidden;}
+[data-testid="stDecoration"] {
+    display: none;
+}
+#root > div:nth-child(1) > div > div > div > div > section > div {
+    padding-top: 0rem;
+}
+
+section.stMain .block-container {
+    padding-top: 0rem;
+    z-index: 1;
+}
+
+/* Main container */
+.stApp {
+    background-color: var(--background);
+    height: auto;
+    overflow: visible;
+}
+
+/* Widgets */
+.stSelectbox,
+.stTextInput,
+.stNumberInput {
+    background-color: var(--card);
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+}
+
+.stMarkdown {
+    color: var(--primary);
+}
+
+/* Code block styling */
+.stCodeBlock {
+    max-height: 400px;
+    overflow-y: auto !important;
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+    background-color: var(--background-shaded);
+}
+
+/* Custom scrollbar for code blocks */
+.stCodeBlock::-webkit-scrollbar {
+    width: 8px;
+    height: 8px;
+}
+
+.stCodeBlock::-webkit-scrollbar-track {
+    background: var(--background-shaded);
+    border-radius: 4px;
+}
+
+.stCodeBlock::-webkit-scrollbar-thumb {
+    background: var(--accent);
+    border-radius: 4px;
+}
+
+.stCodeBlock::-webkit-scrollbar-thumb:hover {
+    background: #e13d68;
+}
+
+
+
+.scrollable-code-container {
+    height: 600px;
+    overflow-y: auto;
+    border: 1px solid var(--border);
+    padding: 15px;
+    border-radius: 5px;
+    background-color: var(--background-shaded);
+}
+
+.scrollable-code-container pre {
+    margin: 0;
+    white-space: pre;
+    overflow-x: auto;
+    font-family: monospace;
+}
+
+.scrollable-code-container code {
+    display: block;
+    min-width: 100%;
+    padding: 0;
+    tab-size: 4;
+}
+
+/* Add custom scrollbar styling for code containers */
+.scrollable-code-container::-webkit-scrollbar {
+    width: 8px;
+    height: 8px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-track {
+    background: var(--background-shaded);
+    border-radius: 4px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-thumb {
+    background: var(--accent);
+    border-radius: 4px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-thumb:hover {
+    background: #e13d68;
+}
+
+/* Style for expander */
+.streamlit-expanderHeader {
+    background-color: var(--card);
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+    padding: 0.5rem !important;
+}
+
+.streamlit-expanderHeader:hover {
+    border-color: var(--accent);
+}
+
+/* Style for expander content */
+.streamlit-expanderContent {
+    background-color: var(--background-shaded);
+    border: 1px solid var(--border);
+    border-radius: 0 0 0.4rem 0.4rem;
+    margin-top: -1px;
+    padding: 0.5rem !important;
+}
+
+/* Style for st.code() blocks */
+.stCode {
+    max-height: 600px;
+    overflow-y: auto;
+    background-color: var(--background-shaded) !important;
+    border: 1px solid var(--border) !important;
+    border-radius: 5px !important;
+}
+
+.stCode pre {
+    background-color: var(--background-shaded) !important;
+}
+
+div[data-testid="InputInstructions"] {
+    visibility: hidden;
+}

app.py

@@ -35,7 +35,7 @@ class WebUI:
         Initialize the WebUI with environment variables and session state.
         """
         self.env_vars = self.load_env_variables()
-        self.project_root = Path(__file__).parent.parent.parent
+        self.project_root = Path(__file__).parent
         self.config_session_state()
         self.setup_page()
 
@@ -219,7 +219,7 @@ class WebUI:
         Returns:
             list: List of example files with their paths.
         """
-        package_root = Path(__file__).parent.parent
+        package_root = Path(__file__).parent / "aide"
         example_dir = package_root / "example_tasks" / "house_prices"
 
         if not example_dir.exists():

requirements.txt

@@ -0,0 +1,94 @@
+# AIDE requirements
+black==24.3.0
+funcy==2.0
+humanize==4.8.0
+jsonschema==4.19.2
+numpy==1.26.2
+openai>=1.3.5
+anthropic>=0.20.0
+pandas==2.1.4
+pytest==7.4.3
+requests==2.32.2
+scikit-learn==1.5.0
+rich==13.7.0
+scipy==1.11.4
+dataclasses_json>=0.6.4
+omegaconf>=2.3.0
+loguru==0.7.2
+shutup>=0.2.0
+tqdm==4.66.3
+coolname>=2.2.0
+igraph>=0.11.3
+genson>=1.2.0
+
+# agent requirements (packages that the agent might need)
+torch
+torchvision
+torchaudio
+torchtext
+
+lightgbm
+matplotlib
+seaborn
+
+# tensorflow
+# tensorflow-io
+# tensorflow_hub
+# tf-keras
+# tensorflow_decision_forests
+# keras
+# keras-cv
+# keras-nlp
+
+gensim
+
+scikit-image
+opencv-python
+scipy
+scikit-learn
+biopython
+imbalanced-learn
+h5py
+biopython
+numba
+arrow
+markovify
+imgaug
+scikit-optimize
+plotly
+hyperopt
+bayesian-optimization
+imagecodecs
+hmmlearn
+bayespy==0.5.1
+sklearn-pandas
+tensorpack
+sentencepiece
+librosa
+ipykernel
+ipython
+nbformat
+kornia
+Pillow
+pyparsing
+pytz
+PyYAML
+tqdm
+fastai
+gym
+optuna
+transformers
+datasets==2.1.0
+torchmetrics
+pytorch-lightning
+sympy
+timm
+torchinfo
+pdf2image
+PyPDF
+pyocr
+pyarrow
+xlrd
+backoff
+streamlit==1.40.2
+python-dotenv

style.css

@@ -0,0 +1,173 @@
+.stVerticalBlock {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+
+.block-container {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+
+[data-testid="stVerticalBlock"] {
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+.st-emotion-cache-40pabj{
+    padding-top: 0rem;
+    padding-bottom: 0rem;
+}
+/* Make the toolbar transparent */
+.st-emotion-cache-18ni7ap {
+    pointer-events: none;
+    background: rgb(255 255 255 / 0%)
+    }
+
+.stAppHeader {
+    display: none;
+}
+
+header {visibility: hidden;}
+#MainMenu {visibility: hidden;}
+[data-testid="stDecoration"] {
+    display: none;
+}
+#root > div:nth-child(1) > div > div > div > div > section > div {
+    padding-top: 0rem;
+}
+
+section.stMain .block-container {
+    padding-top: 0rem;
+    z-index: 1;
+}
+
+/* Main container */
+.stApp {
+    background-color: var(--background);
+    height: auto;
+    overflow: visible;
+}
+
+/* Widgets */
+.stSelectbox,
+.stTextInput,
+.stNumberInput {
+    background-color: var(--card);
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+}
+
+.stMarkdown {
+    color: var(--primary);
+}
+
+/* Code block styling */
+.stCodeBlock {
+    max-height: 400px;
+    overflow-y: auto !important;
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+    background-color: var(--background-shaded);
+}
+
+/* Custom scrollbar for code blocks */
+.stCodeBlock::-webkit-scrollbar {
+    width: 8px;
+    height: 8px;
+}
+
+.stCodeBlock::-webkit-scrollbar-track {
+    background: var(--background-shaded);
+    border-radius: 4px;
+}
+
+.stCodeBlock::-webkit-scrollbar-thumb {
+    background: var(--accent);
+    border-radius: 4px;
+}
+
+.stCodeBlock::-webkit-scrollbar-thumb:hover {
+    background: #e13d68;
+}
+
+
+
+.scrollable-code-container {
+    height: 600px;
+    overflow-y: auto;
+    border: 1px solid var(--border);
+    padding: 15px;
+    border-radius: 5px;
+    background-color: var(--background-shaded);
+}
+
+.scrollable-code-container pre {
+    margin: 0;
+    white-space: pre;
+    overflow-x: auto;
+    font-family: monospace;
+}
+
+.scrollable-code-container code {
+    display: block;
+    min-width: 100%;
+    padding: 0;
+    tab-size: 4;
+}
+
+/* Add custom scrollbar styling for code containers */
+.scrollable-code-container::-webkit-scrollbar {
+    width: 8px;
+    height: 8px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-track {
+    background: var(--background-shaded);
+    border-radius: 4px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-thumb {
+    background: var(--accent);
+    border-radius: 4px;
+}
+
+.scrollable-code-container::-webkit-scrollbar-thumb:hover {
+    background: #e13d68;
+}
+
+/* Style for expander */
+.streamlit-expanderHeader {
+    background-color: var(--card);
+    border: 1px solid var(--border);
+    border-radius: 0.4rem;
+    padding: 0.5rem !important;
+}
+
+.streamlit-expanderHeader:hover {
+    border-color: var(--accent);
+}
+
+/* Style for expander content */
+.streamlit-expanderContent {
+    background-color: var(--background-shaded);
+    border: 1px solid var(--border);
+    border-radius: 0 0 0.4rem 0.4rem;
+    margin-top: -1px;
+    padding: 0.5rem !important;
+}
+
+/* Style for st.code() blocks */
+.stCode {
+    max-height: 600px;
+    overflow-y: auto;
+    background-color: var(--background-shaded) !important;
+    border: 1px solid var(--border) !important;
+    border-radius: 5px !important;
+}
+
+.stCode pre {
+    background-color: var(--background-shaded) !important;
+}
+
+div[data-testid="InputInstructions"] {
+    visibility: hidden;
+}