forked from satwikkansal/wtfpython
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathnotebook_generator.py
More file actions
354 lines (296 loc) · 12 KB
/
Copy pathnotebook_generator.py
File metadata and controls
354 lines (296 loc) · 12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
"""
An inefficient monolithic piece of code that'll generate jupyter notebook
from the projects main README.
PS: If you are a recruiter, please don't judge me by this piece of code. I wrote it
in hurry. I know this is messy and can be simplified, but I don't want to change it
much because it just works.
Simplifictions and improvements through patches are more than welcome however :)
#TODOs
- CLI arguments for running this thing
- Add it to prepush hook
- Add support for skip comments, to skip examples that are not meant for notebook environment.
- Use templates?
"""
import json
import os
import pprint
fpath = os.path.join(os.path.dirname( __file__ ), '..', 'README.md')
examples = []
# The globals
current_example = 1
sequence_num = 1
current_section_name = ""
STATEMENT_PREFIXES = ["...", ">>> ", "$ "]
HOSTED_NOTEBOOK_INSTRUCTIONS = """
## Hosted notebook instructions
This is just an experimental attempt of browsing wtfpython through jupyter notebooks. Some examples are read-only because,
- they either require a version of Python that's not supported in the hosted runtime.
- or they can't be reproduced in the notebook envrinonment.
The expected outputs are already present in collapsed cells following the code cells. The Google colab provides Python2 (2.7) and Python3 (3.6, default) runtimes. You can switch among these for Python2 specific examples. For examples specific to other minor versions, you can simply refer to collapsed outputs (it's not possible to control the minor version in hosted notebooks as of now). You can check the active version using
```py
>>> import sys
>>> sys.version
# Prints out Python version here.
```
That being said, most of the examples do work as expected. If you face any trouble, feel free to consult the original content on wtfpython and create an issue in the repo. Have fun!
---
"""
def generate_code_block(statements, output):
global sequence_num
result = {
"type": "code",
"sequence_num": sequence_num,
"statements": statements,
"output": output
}
sequence_num += 1
return result
def generate_markdown_block(lines):
global sequence_num
result = {
"type": "markdown",
"sequence_num": sequence_num,
"value": lines
}
sequence_num += 1
return result
def is_interactive_statement(line):
for prefix in STATEMENT_PREFIXES:
if line.lstrip().startswith(prefix):
return True
return False
def parse_example_parts(lines, title, current_line):
parts = {
"build_up": [],
"explanation": []
}
content = [title]
statements_so_far = []
output_so_far = []
next_line = current_line
# store build_up till an H4 (explanation) is encountered
while not (next_line.startswith("#### ")or next_line.startswith('---')):
# Watching out for the snippets
if next_line.startswith("```py"):
# It's a snippet, whatever found until now is text
is_interactive = False
output_encountered = False
if content:
parts["build_up"].append(generate_markdown_block(content))
content = []
next_line = next(lines)
while not next_line.startswith("```"):
if is_interactive_statement(next_line):
is_interactive = True
if (output_so_far):
parts["build_up"].append(generate_code_block(statements_so_far, output_so_far))
statements_so_far, output_so_far = [], []
statements_so_far.append(next_line)
else:
# can be either output or normal code
if is_interactive:
output_so_far.append(next_line)
elif output_encountered:
output_so_far.append(next_line)
else:
statements_so_far.append(next_line)
next_line = next(lines)
# Snippet is over
parts["build_up"].append(generate_code_block(statements_so_far, output_so_far))
statements_so_far, output_so_far = [], []
next_line = next(lines)
else:
# It's a text, go on.
content.append(next_line)
next_line = next(lines)
# Explanation encountered, save any content till now (if any)
if content:
parts["build_up"].append(generate_markdown_block(content))
# Reset stuff
content = []
statements_so_far, output_so_far = [], []
# store lines again until --- or another H3 is encountered
while not (next_line.startswith("---") or
next_line.startswith("### ")):
if next_line.lstrip().startswith("```py"):
# It's a snippet, whatever found until now is text
is_interactive = False
if content:
parts["explanation"].append(generate_markdown_block(content))
content = []
next_line = next(lines)
while not next_line.lstrip().startswith("```"):
if is_interactive_statement(next_line):
is_interactive = True
if (output_so_far):
parts["explanation"].append(generate_code_block(statements_so_far, output_so_far))
statements_so_far, output_so_far = [], []
statements_so_far.append(next_line)
else:
# can be either output or normal code
if is_interactive:
output_so_far.append(next_line)
else:
statements_so_far.append(next_line)
next_line = next(lines)
# Snippet is over
parts["explanation"].append(generate_code_block(statements_so_far, output_so_far))
statements_so_far, output_so_far = [], []
next_line = next(lines)
else:
# It's a text, go on.
content.append(next_line)
next_line = next(lines)
# All done
if content:
parts["explanation"].append(generate_markdown_block(content))
return next_line, parts
def remove_from_beginning(tokens, line):
for token in tokens:
if line.lstrip().startswith(token):
line = line.replace(token, "")
return line
def inspect_and_sanitize_code_lines(lines):
tokens_to_remove = STATEMENT_PREFIXES
result = []
is_print_present = False
for line in lines:
line = remove_from_beginning(tokens_to_remove, line)
if line.startswith("print ") or line.startswith("print("):
is_print_present = True
result.append(line)
return is_print_present, result
def convert_to_cells(cell_contents, read_only):
cells = []
for stuff in cell_contents:
if stuff["type"] == "markdown":
# todo add metadata later
cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": stuff["value"]
}
)
elif stuff["type"] == "code":
if read_only:
# Skip read only
# TODO: Fix
cells.append(
{
"cell_type": "markdown",
"metadata": {},
"source": ["```py\n"] + stuff["statements"] + ["```\n"] + ["```py\n"] + stuff['output'] + ["```\n"]
}
)
continue
is_print_present, sanitized_code = inspect_and_sanitize_code_lines(stuff["statements"])
if is_print_present:
cells.append(
{
"cell_type": "code",
"metadata": {
"collapsed": True,
},
"execution_count": None,
"outputs": [{
"name": "stdout",
"output_type": "stream",
"text": stuff["output"]
}],
"source": sanitized_code
}
)
else:
cells.append(
{
"cell_type": "code",
"execution_count": None,
"metadata": {
"collapsed": True
},
"outputs": [{
"data": {
"text/plain": stuff["output"]
},
"output_type": "execute_result",
"metadata": {},
"execution_count": None
}],
"source": sanitized_code
}
)
return cells
def convert_to_notebook(pre_examples_content, parsed_json, post_examples_content):
result = {
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 2
}
notebook_path = "wtf.ipynb"
result["cells"] += convert_to_cells([generate_markdown_block(pre_examples_content)], False)
for example in parsed_json:
parts = example["parts"]
build_up = parts.get("build_up")
explanation = parts.get("explanation")
read_only = example.get("read_only")
if build_up:
result["cells"] += convert_to_cells(build_up, read_only)
if explanation:
result["cells"] += convert_to_cells(explanation, read_only)
result["cells"] += convert_to_cells([generate_markdown_block(post_examples_content)], False)
#pprint.pprint(result, indent=2)
with open(notebook_path, "w") as f:
json.dump(result, f, indent=2)
with open(fpath, 'r+', encoding="utf-8") as f:
lines = iter(f.readlines())
line = next(lines)
result = []
pre_examples_phase = True
pre_stuff = []
post_stuff = []
try:
while True:
if line.startswith("## "):
pre_examples_phase = False
# A section is encountered
current_section_name = line.replace("## ", "").strip()
section_text = []
line = next(lines)
# Until a new section is encountered
while not (line.startswith("## ") or line.startswith("# ")):
# check if it's a H3
if line.startswith("### "):
# An example is encountered
title_line = line
line = next(lines)
read_only = False
while line.strip() == "" or line.startswith('<!--'):
#TODO: Capture example ID here using regex.
if '<!-- read-only -->' in line:
read_only = True
line = next(lines)
example_details = {
"id": current_example,
"title": title_line.replace("### ", ""),
"section": current_section_name,
"read_only": read_only
}
line, example_details["parts"] = parse_example_parts(lines, title_line, line)
result.append(example_details)
current_example += 1
else:
section_text.append(line)
line = next(lines)
else:
if pre_examples_phase:
pre_stuff.append(line)
else:
post_stuff.append(line)
line = next(lines)
except StopIteration as e:
#pprint.pprint(result, indent=2)
pre_stuff.append(HOSTED_NOTEBOOK_INSTRUCTIONS)
result.sort(key = lambda x: x["read_only"])
convert_to_notebook(pre_stuff, result, post_stuff)