review_config:
  # User configuration for the review
  # - benchmark - use the user config from the benchmark reviews
  # - <value> - use the value directly
  user_config:
    enable_ai_review: true
    enable_rule_comments: false

    enable_complexity_comments: benchmark
    enable_docstring_comments: benchmark
    enable_security_comments: benchmark
    enable_tests_comments: benchmark
    enable_comment_suggestions: benchmark

    enable_approvals: true

  ai_review_config:
    # The model responses to use for the experiment
    # - benchmark - use the model responses from the benchmark reviews
    # - llm - call the language model to generate responses
    model_responses:
      comments_model: benchmark
      comment_validation_model: benchmark
      comment_suggestion_model: benchmark
      complexity_model: benchmark
      docstrings_model: benchmark
      security_model: benchmark
      tests_model: benchmark

# The pull request dataset to run the experiment on
pull_request_dataset:
- https://github.com/mraniki/iamlistening/pull/294
- https://github.com/gdsfactory/gplugins/pull/373
- https://github.com/Anush008/fastembed-rs/pull/48
- https://github.com/mraniki/tt/pull/1435
- https://github.com/kloudlite/operator/pull/172
- https://github.com/mraniki/iamlistening/pull/293
- https://github.com/mraniki/iamlistening/pull/292
- https://github.com/mraniki/cefi/pull/434
- https://github.com/kloudlite/operator/pull/171
- https://github.com/usama-maxenius/image-editor/pull/62
- https://github.com/mraniki/tt/pull/1434
- https://github.com/mraniki/dxsp/pull/614
- https://github.com/albumentations-team/albumentations/pull/1637
- https://github.com/erxes/erxes/pull/5119
- https://github.com/mraniki/cefi/pull/433
- https://github.com/Quarticai/QuarticSDK/pull/358
- https://github.com/mraniki/cefi/pull/432
- https://github.com/tpaviot/pythonocc-core/pull/1311
- https://github.com/lightning-bot/Lightning/pull/144
- https://github.com/ignition-api/8.1/pull/265
- https://github.com/fairdataihub/fairdataihub.org/pull/616
# - https://github.com/suttacentral/suttacentral/pull/3122
# - https://github.com/jquagga/ttt/pull/30
# - https://github.com/jquagga/ttt/pull/29
# - https://github.com/Harrytimbog/Peer-Pal/pull/22
# - https://github.com/bengosney/cerberus/pull/790
# - https://github.com/Harrytimbog/Peer-Pal/pull/21
# - https://github.com/mraniki/cefi/pull/431
# - https://github.com/bengosney/cerberus/pull/789
# - https://github.com/bengosney/cerberus/pull/788
# - https://github.com/jmcerrejon/PiKISS/pull/214
# - https://github.com/mraniki/dxsp/pull/613
# - https://github.com/mraniki/cefi/pull/430
# - https://github.com/mraniki/cefi/pull/429
# - https://github.com/gdsfactory/gdsfactory/pull/2658
# - https://github.com/Bilbottom/sql-learning-materials/pull/7
# - https://github.com/mraniki/cefi/pull/428
# - https://github.com/KonScanner/synthr-farming/pull/1
# - https://github.com/rtk-rnjn/algorithms/pull/78
# - https://github.com/malayilneil/lab04/pull/1
# - https://github.com/nbhirud/system_update/pull/6
# - https://github.com/mraniki/cefi/pull/427
# - https://github.com/Kilo59/ruff-sync/pull/16
# - https://github.com/jquagga/ttt/pull/27
# - https://github.com/alexiusstrauss/CryptoTrendAnalyzer/pull/10
# - https://github.com/jquagga/ttt/pull/25
# - https://github.com/mraniki/tt/pull/1425
# - https://github.com/albumentations-team/albumentations_stats/pull/1
# - https://github.com/jquagga/ttt/pull/24
# - https://github.com/jsugg/retry-on/pull/1
# - https://github.com/strawberry-graphql/strawberry/pull/3442
# - https://github.com/jquagga/ttt/pull/23
# - https://github.com/jquagga/ttt/pull/22
# - https://github.com/jquagga/ttt/pull/21
# - https://github.com/jquagga/ttt/pull/20
# - https://github.com/Kilo59/ruff-sync/pull/14
# - https://github.com/jquagga/ttt/pull/19
# - https://github.com/jquagga/ttt/pull/18
# - https://github.com/jquagga/ttt/pull/17
# - https://github.com/brendancsmith/diffbot-kg/pull/3
# - https://github.com/2lambda123/StenaIT-stenajs-webui/pull/1
# - https://github.com/jkool702/openwrt/pull/24
# - https://github.com/KevinNitroG/VNULIB-Downloader/pull/28
# - https://github.com/CPUT-DEVS/devpost-hackathon/pull/15
# - https://github.com/code-Harsh247/FRSS-project/pull/34
# - https://github.com/code-Harsh247/FRSS-project/pull/33
# - https://github.com/kurianbenoy/samam-ml-verification/pull/1
# - https://github.com/alexiusstrauss/CryptoTrendAnalyzer/pull/8
# - https://github.com/mraniki/dxsp/pull/612
# - https://github.com/alexiusstrauss/CryptoTrendAnalyzer/pull/7
# - https://github.com/alexiusstrauss/CryptoTrendAnalyzer/pull/6
# - https://github.com/neurodatascience/cohort_creator/pull/207
# - https://github.com/albumentations-team/albumentations-demo/pull/12
# - https://github.com/mraniki/cefi/pull/426
# - https://github.com/alexiusstrauss/CryptoTrendAnalyzer/pull/5
# - https://github.com/jkool702/openwrt/pull/23
# - https://github.com/jkool702/openwrt/pull/22
# - https://github.com/ynvtlmr/intergenerational-family-code/pull/108
# - https://github.com/PythonFreeCourse/lms/pull/390
# - https://github.com/jquagga/ttt/pull/16
# - https://github.com/PythonFreeCourse/lms/pull/389
# - https://github.com/PythonFreeCourse/lms/pull/389
# - https://github.com/PythonFreeCourse/lms/pull/389
# - https://github.com/jquagga/ttt/pull/13
# - https://github.com/Speccy-Rom/Leetcode_aka_speccy-rom/pull/293
# - https://github.com/Bilbottom/sql-learning-materials/pull/5
# - https://github.com/mraniki/cefi/pull/425
# - https://github.com/approvals/Approvals.NodeJS/pull/173
# - https://github.com/gdsfactory/gdsfactory/pull/2657
# - https://github.com/mraniki/tt/pull/1420
# - https://github.com/vibikerski/trackingtasks/pull/2
# - https://github.com/yaitoo/sqle/pull/30
# - https://github.com/jquagga/ttt/pull/12
# - https://github.com/Mesteriis/test-repo/pull/4
# - https://github.com/Mesteriis/test-repo/pull/3
# - https://github.com/Mesteriis/test-repo/pull/2
# - https://github.com/Mesteriis/test-repo/pull/1
# - https://github.com/letsdoitnowus/planium-backend/pull/34
# - https://github.com/code-Harsh247/FRSS-project/pull/32
# - https://github.com/letsdoitnowus/planium-backend/pull/33

# Questions to ask to label the review comments
review_comment_labels:
- label: correct
  question: Is this comment correct?
- label: helpful
  question: Is this comment helpful?
- label: comment-type
  question: Is the comment type correct?
- label: comment-area
  question: Is the comment area correct?

# Benchmark reviews generated by running
#   python -m scripts.experiment benchmark <experiment_name>
benchmark_reviews:
- dataset_pull_request: https://github.com/jquagga/ttt/pull/12
  review_pull_request: https://github.com/sourcery-ai-experiments/ttt/pull/25