pythoness/pythoness/pythoness_module.py at main · sanath-upadhya/pythoness

History

487 lines (419 loc) · 17.9 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

import inspect

import io

import json

import logging

import re

import sys

import textwrap

import openai

import sqlite3

import traceback

from hypothesis import example, given

from hypothesis.strategies import *

from collections.abc import Generator

import ast_comments as ast

import __main__ as main

from functools import wraps

from typing import Callable, Tuple

debug_print = False

def is_interactive():

if not hasattr(main, "__file__"):

# executed interactively (e.g. at the CLI or in a Jupyter notebook)

return True

else:

# executed non-interactively (executing a script)

return False

def is_type_compatible(f: Callable, g: Callable) -> bool:

f_sig = inspect.signature(f)

g_sig = inspect.signature(g)

# Check number of parameters

if len(f_sig.parameters) != len(g_sig.parameters):

if debug_print:

print("mismatch in number of parameters")

return False

# Check parameter types

for f_param, g_param in zip(f_sig.parameters.values(), g_sig.parameters.values()):

f_type = f_param.annotation

g_type = g_param.annotation

# If the second function's type is missing or Any, and the first function's type is not, they are compatible.

if g_type is inspect.Parameter.empty or g_type is type(None):

continue

elif f_type is inspect.Parameter.empty or f_type is type(None):

# For now, we consider this to be compatible.

continue

# if not issubclass(type(None), g_type):

# return False

if not issubclass(g_type, f_type) and not issubclass(f_type, g_type):

if debug_print:

print(f"subclass mismatch: f: {f_type}, g: {g_type}")

return False

# Check return type

f_return_type = f_sig.return_annotation

g_return_type = g_sig.return_annotation

# If the second function's return type is missing or Any, and the first function's return type is not, they are compatible.

if g_return_type is inspect.Parameter.empty or g_return_type is type(None):

return True

elif f_return_type is inspect.Parameter.empty or f_return_type is type(None):

if issubclass(type(None), g_return_type):

return True

else:

if debug_print:

print("subclass issue with return types")

return False

if not issubclass(g_return_type, f_return_type) and not issubclass(

f_return_type, g_return_type

if debug_print:

print("second subclass issue with return types")

return False

return True

class CodeDatabase:

def __init__(self, db_file):

self.db_file = db_file

self.connection = sqlite3.connect(db_file)

self.cursor = self.connection.cursor()

self.create_table()

def create_table(self):

self.cursor.execute(

"""

CREATE TABLE IF NOT EXISTS prompt_code (

id INTEGER PRIMARY KEY AUTOINCREMENT,

prompt TEXT NOT NULL,

code TEXT NOT NULL

)

"""

)

self.cursor.execute(

"""

CREATE INDEX IF NOT EXISTS index_prompt ON prompt_code (prompt)

"""

)

self.connection.commit()

def insert_code(self, prompt, code):

self.cursor.execute(

"INSERT INTO prompt_code (prompt, code) VALUES (?, ?)", (prompt, code)

)

self.connection.commit()

def get_code(self, prompt):

self.cursor.execute("SELECT code FROM prompt_code WHERE prompt = ?", (prompt,))

row = self.cursor.fetchone()

if row is not None:

return row[0]

else:

return None

def close(self):

self.connection.close()

def complete(user_prompt: str, history: list) -> str:

"""Initiates a conversation with ChatGPT with the users description of the function.

Returns the output generated by ChatGPT.

"""

initial_timeout = 30

while True:

try:

history.append({"role": "user", "content": user_prompt})

completion = openai.ChatCompletion.create(

# For now, hard code

model="gpt-4", # args["llm"],

request_timeout=initial_timeout, # args["timeout"],

messages= history,

)

history.append({"role": "assistant", "content": completion.choices[0].message.content})

return completion.choices[0].message.content

except openai.error.AuthenticationError:

print("You need an OpenAI key to use this tool.")

print(

"You can get a key here: https://platform.openai.com/account/api-keys"

)

print("Set the environment variable OPENAI_API_KEY to your key value.")

print(

"If OPENAI_API_KEY is already correctly set, you may have exceeded your usage or rate limit."

)

sys.exit(1)

except openai.error.Timeout:

# Exponential growth.

initial_timeout *= 2

def spec(

string,

replace=False,

tests=None,

max_retries=3,

verbose=False,

min_confidence=0.7,

output=False,

regenerate=False,

def decorator(func):

cached_function = None

cdb = CodeDatabase("pythoness-cache.db")

@wraps(func)

def wrapper(*args, **kwargs):

nonlocal cdb, cached_function

# PROPOSED FEATURE: we could have a flag (lazy=True) control whether

# we wait until invocation to try to synthesize functions

# or (lazy=False) which would speculatively attempt to

# resolve all spec functions asynchronously (as futures).

if regenerate:

# Clear the cached function if we are regenerating.

cached_function = None

# If we've already built this function and cached it, just

# run it.

if cached_function:

return cached_function(*args, **kwargs)

# We need to generate a function from the spec.

# We populate the prompt with the function's name, argument name and types, and the return type.

function_name = func.__name__

arg_types = []

for arg_name, arg_value in zip(func.__code__.co_varnames, args):

arg_types.append(

(arg_name, type(arg_value))

) # FIXME: use annotations if available

for kwarg_name, kwarg_value in kwargs.items():

arg_types.append(

(

kwarg_name,

type(kwarg_value),

)

) # FIXME: use annotations if available

return_type = func.__annotations__.get("return", None)

prompt = f"""

Produce a JSON object with code for a Python function

named {function_name} that performs the following task as

a field \"code\". Report your confidence that this code

performs the task as a number between 0 and 1, as a field

\"confidence\". Only produce output that can be parsed as

JSON.

Task:

{textwrap.dedent(string)}

Include a docstring containing the task description above

(without the word "Task:"). The function should be

entirely self-contained, with all imports, code, and data

required for its functionality. """

if tests:

final_tests = []

for t in tests:

if isinstance(t, tuple):

final_tests.append(t[1])

elif isinstance(t, str):

final_tests.append(t)

else:

pass

test_string = "\n ".join(final_tests)

prompt += f"""

The function should pass the following tests:

{test_string}

"""

prompt += f"""

The function should have the following argument types and return type:

Arguments: {arg_types}

Return type: {return_type}

"""

if verbose:

print("[Pythoness] Prompt:\n", prompt)

# See if we already have code corresponding to that prompt in the database.

if regenerate:

# Force regeneration by ignoring any existing code in the database.

function_def = None

else:

function_def = cdb.get_code(prompt)

if verbose and function_def:

print("[Pythoness] retrieved function from database:\n", function_def)

# We have previously loaded the function. Just execute it and return.

if function_def:

compiled = compile(function_def, "<string>", "exec")

exec(compiled, globals())

fn = globals()[function_name]

return fn(*args, **kwargs)

# Keep track of basic (anonymous) statistics.

stats = {}

stats["spec"] = string

stats["function_name"] = function_name

stats["tests_provided"] = json.dumps(tests)

stats["num_tests_failed"] = 0

stats["retries"] = 0

stats["successes"] = 0

stats["parse_failures"] = 0

stats["execution_failures"] = 0

stats["below_confidence_level"] = 0

stats["compilation_failures"] = 0

stats["type_incompatibility_failures"] = 0

stats["test_failures"] = 0

stats["min_confidence"] = min_confidence

history = []

failing_tests = set()

while stats["retries"] < max_retries:

stats["retries"] += 1

if verbose:

print(f"[Pythoness] Attempt number {stats['retries']}.")

# Retry until success.

if not function_def:

result = complete(prompt, history)

try:

the_json = json.loads(result)

except:

# JSON parse failure: retry.

stats["parse_failures"] += 1

if verbose:

print("[Pythoness] JSON parsing failed.")

continue

function_def = the_json["code"]

confidence = float(the_json["confidence"])

if verbose:

print("[Pythoness] Synthesized function\n", function_def)

print("[Pythoness] Confidence:", confidence)

if confidence < min_confidence:

stats["confidence_failures"] += 1

if verbose:

print(

f"[Pythoness] Confidence level {confidence} too low (below {min_confidence})."

)

continue

# Try to compile the function

try:

compiled = compile(function_def, "<string>", "exec")

except:

# Compilation failed: retry.

stats["compilation_failures"] += 1

if verbose:

print("[Pythoness] Compilation failed.")

function_def = None

continue

# If we get here, we can run the function and use it going forwards.

try:

exec(compiled, globals())

except:

if verbose:

print("[Pythoness] Executing the function failed.")

stats["execution_failures"] += 1

function_def = None

continue

fn = globals()[function_name]

if not is_type_compatible(func, fn):

stats["type_incompatibility_failures"] += 1

# Function types don't validate. Retry.

if verbose:

print(

"[Pythoness] The generated function is incompatible with the spec."

)

function_def = None

continue

# Validate tests.

if tests:

for t in tests:

try:

if isinstance(t, tuple):

compiled_hypothesis_test = create_hypothesis_test(t)

exec(compiled_hypothesis_test,globals())

else:

if not eval(t):

failing_tests.add(t)

except AssertionError:

exc_type, exc_value, exc_tb = sys.exc_info()

tb = traceback.TracebackException(exc_type, exc_value, exc_tb)

exception_info = tb.format_exception_only()

line_number = 0

falsifying_example = get_falsifying_example(exception_info)

for exception_line in exception_info:

logging.DEBUG(str(line_number) + " "+ exception_line)

line_number += 1

print("Falsifying example is "+ falsifying_example)

if isinstance(t,tuple):

#Convert the dict in first element of tuple to string

#and add this newly modified tuple to failing_tests

new_l = list(t)

string_input = str(t[0])

new_l[0] = string_input

new_t = tuple(new_l)

failing_tests.add(new_t)

else:

failing_tests.add(t)

except:

raise Exception(

f"This test failed to execute properly: {t}"

)

if len(failing_tests) > 0:

stats["test_failures"] += 1

stats["num_tests_failed"] += len(failing_tests)

# At least one test failed. Retry.

if verbose:

print(f"[Pythoness] Tests failed: {failing_tests}")

function_def = None

continue

stats["successes"] += 1

logging.info(json.dumps(stats))

# Validated. Cache the function and persist it.

cached_function = fn

cdb.insert_code(prompt, function_def)

if output: # or is_interactive():

print(function_def, file=sys.stdout)

# If selected, replace the function definition

# in the file.

if replace:

import inspect

frame = inspect.currentframe()

frame = frame.f_back

file_name = frame.f_code.co_filename

with open(file_name, "r") as file:

source = file.read()

tree = ast.parse(source)

# Find the function with the given name and replace it with the new function.

for node in ast.walk(tree):

if (

isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))

and node.name == function_name

node_index = tree.body.index(node)

fn_body = ast.parse(function_def).body

tree.body[node_index] = fn_body

new_source = ast.unparse(tree)

# Update the file.

with open(file_name, "w") as f:

f.write(new_source)

return cached_function(*args, **kwargs)

# If we got here, we had too many retries.

logging.info(json.dumps(stats))

if failing_tests:

raise Exception(

f"Maximum number of retries exceeded ({max_retries}).\nFailing tests: {failing_tests}"

)

else:

raise Exception(f"Maximum number of retries exceeded ({max_retries}).")

return wrapper

return decorator

def get_falsifying_example(exception_info: Generator[str, None, None]) -> str:

"""Obtain the values of the parameters for which the hypothesis test is failing.

"""

assertion_error = False

result = ''

for exception_line in exception_info:

if "AssertionError" in exception_line:

assertion_error = True

break

if assertion_error:

input_start = False

for exception_line in exception_info:

if ")" in exception_line:

input_start = False

continue

if "Falsifying example:" in exception_line:

input_start = True

continue

if input_start:

result = result + exception_line.strip()

return result

def create_hypothesis_test(t):

if isinstance(t[0],dict):

assertion = t[1]

given_input = ",".join(t[0].values())

parameter_input = ",".join(list(t[0].keys()))

hypothesis_test = f"""

@given({given_input})

def test({parameter_input}):

assert({assertion})

test()

"""

else:

raise Exception(f"The following test does not have a dictionary in it ({t}). Please use correct syntax")

return compile(hypothesis_test,"<string>","exec")

logging.basicConfig(

filename="pythoness.log", encoding="utf-8", format="%(message)s", level=logging.DEBUG

)

FilesExpand file tree

pythoness_module.py

Latest commit

History

pythoness_module.py

File metadata and controls