Skip to content

Improve lexer input buffering and Unix generator builds #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Berkan Karakuş (berkankarakus) wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Improve lexer input buffering and Unix generator builds #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8622f57
perf: reduce lexer input buffer compaction
berkankarakus Nov 8, 2026
644cb82
perf: use circular queue for lexer input
berkankarakus Nov 8, 2026
2c6c5f0
fix(build): run generators through dotnet on Unix
berkankarakus Nov 8, 2026
382ca9f
fix(build): roll generator runtime on Unix
berkankarakus Nov 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions SqlScriptDom/GenerateFiles.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,8 @@
</ItemGroup>
</Target>
<Target Name="CreateAST" DependsOnTargets="AfterResolveReferences;GSqlTokenTypesCompile" BeforeTargets="CoreCompile" Inputs="$(SQLSCRIPTDOM)\Parser\TSql\Ast.xml;" Outputs="$(CsGenIntermediateOutputPath)Ast.cs;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs">
<Exec Condition="'$(SqlDomNuget)'!='true' And '$(OS)' != 'Windows_NT'" Command="&quot;$(BuildToolsPath)/astgen&quot; &quot;$(SQLSCRIPTDOM)/Parser/TSql/Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; " />
<Exec Condition="'$(SqlDomNuget)'=='true' And '$(OS)' != 'Windows_NT'" Command="&quot;$(BuildToolsPath)/astgen&quot; &quot;$(SQLSCRIPTDOM)/Parser/TSql/Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; -test &quot;$(CsGenIntermediateOutputPath)\OverrideSqlScriptGeneratorVisitor.codegen.cs&quot; &quot;" />
<Exec Condition="'$(SqlDomNuget)'!='true' And '$(OS)' != 'Windows_NT'" Command="DOTNET_ROLL_FORWARD=Major dotnet &quot;$(BuildToolsPath)/astgen.dll&quot; &quot;$(SQLSCRIPTDOM)/Parser/TSql/Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; " />
<Exec Condition="'$(SqlDomNuget)'=='true' And '$(OS)' != 'Windows_NT'" Command="DOTNET_ROLL_FORWARD=Major dotnet &quot;$(BuildToolsPath)/astgen.dll&quot; &quot;$(SQLSCRIPTDOM)/Parser/TSql/Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; -test &quot;$(CsGenIntermediateOutputPath)\OverrideSqlScriptGeneratorVisitor.codegen.cs&quot; &quot;" />
<Exec Condition="'$(SqlDomNuget)'!='true' And '$(OS)' == 'Windows_NT'" Command="&quot;$(BuildToolsPath)\AstGen.exe&quot; &quot;$(SQLSCRIPTDOM)\Parser\TSql\Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; " />
<Exec Condition="'$(SqlDomNuget)'=='true' And '$(OS)' == 'Windows_NT'" Command="&quot;$(BuildToolsPath)\AstGen.exe&quot; &quot;$(SQLSCRIPTDOM)\Parser\TSql\Ast.xml&quot; &quot;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)TSqlConcreteFragmentVisitor.codegen.cs&quot; &quot;$(CsGenIntermediateOutputPath)Ast.cs&quot; -test &quot;$(CsGenIntermediateOutputPath)\OverrideSqlScriptGeneratorVisitor.codegen.cs&quot; &quot;" />
<ItemGroup>
Expand All @@ -70,8 +70,8 @@
</ItemGroup>
</Target>
<Target Name="GenerateEverything" DependsOnTargets="AfterResolveReferences;GSqlTokenTypesCompile" BeforeTargets="CoreCompile" Inputs="$(SQLSCRIPTDOM)\ScriptDom\SqlServer\Settings\SqlScriptGeneratorOptions.xml" Outputs="$(CsGenIntermediateOutputPath)TSqlTokenTypes.txt;$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptions.cs;$(CsGenIntermediateOutputPath)ScriptGeneratorSupporter.Generated.cs">
<Exec Condition="'$(OS)' != 'Windows_NT'" Command="&quot;$(BuildToolsPath)/ScriptGenSettingsGenerator&quot; &quot;$(SQLSCRIPTDOM)/ScriptDom/SqlServer/Settings/SqlScriptGeneratorOptions.xml&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptions.cs&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptionsAdapter.Generated.cs&quot;" />
<Exec Condition="'$(OS)' != 'Windows_NT'" Command="&quot;$(BuildToolsPath)/TokenListGenerator&quot; &quot;$(CsGenIntermediateOutputPath)TSqlTokenTypes.txt&quot; &gt; &quot;$(CsGenIntermediateOutputPath)ScriptGeneratorSupporter.Generated.cs&quot;" />
<Exec Condition="'$(OS)' != 'Windows_NT'" Command="DOTNET_ROLL_FORWARD=Major dotnet &quot;$(BuildToolsPath)/ScriptGenSettingsGenerator.dll&quot; &quot;$(SQLSCRIPTDOM)/ScriptDom/SqlServer/Settings/SqlScriptGeneratorOptions.xml&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptions.cs&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptionsAdapter.Generated.cs&quot;" />
<Exec Condition="'$(OS)' != 'Windows_NT'" Command="DOTNET_ROLL_FORWARD=Major dotnet &quot;$(BuildToolsPath)/TokenListGenerator.dll&quot; &quot;$(CsGenIntermediateOutputPath)TSqlTokenTypes.txt&quot; &gt; &quot;$(CsGenIntermediateOutputPath)ScriptGeneratorSupporter.Generated.cs&quot;" />
<Exec Condition="'$(OS)' == 'Windows_NT'" Command="&quot;$(BuildToolsPath)\ScriptGenSettingsGenerator.exe&quot; &quot;$(SQLSCRIPTDOM)\ScriptDom\SqlServer\Settings\SqlScriptGeneratorOptions.xml&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptions.cs&quot; &quot;$(CsGenIntermediateOutputPath)SqlScriptGeneratorOptionsAdapter.Generated.cs&quot;" />
<Exec Condition="'$(OS)' == 'Windows_NT'" Command="&quot;$(BuildToolsPath)\TokenListGenerator.exe&quot; &quot;$(CsGenIntermediateOutputPath)TSqlTokenTypes.txt&quot; &gt; &quot;$(CsGenIntermediateOutputPath)ScriptGeneratorSupporter.Generated.cs&quot;" />
<ItemGroup>
Expand All @@ -80,4 +80,4 @@
<Compile Condition="'$(SqlDomNuget)'!='true'" Include="$(CsGenIntermediateOutputPath)TSqlFragmentVisitor.codegen.cs" />
</ItemGroup>
</Target>
</Project>
</Project>
18 changes: 9 additions & 9 deletions SqlScriptDom/antlr/antlr/ByteBuffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ override public void fill(int amount)
// {
syncConsume();
// Fill the buffer sufficiently to hold needed characters
int bytesToRead = (amount + markerOffset) - queue.Count;
int bytesToRead = (amount + markerOffset) - queue.nbrEntries;
int c;

while (bytesToRead > 0)
Expand All @@ -72,15 +72,15 @@ override public void fill(int amount)
for (int i = 0; i < c; i++)
{
// Append the next character
queue.Add(unchecked((char) buf[i]));
queue.append(unchecked((char) buf[i]));
}
bytesToRead -= c;
if (c < BUF_SIZE)
bytesToRead -= c;
if (c < BUF_SIZE)
{
while (bytesToRead-- > 0)
{
queue.Add(CharScanner.EOF_CHAR);
}
while (bytesToRead-- > 0)
{
queue.append(CharScanner.EOF_CHAR);
}
break;
}
}
Expand All @@ -91,4 +91,4 @@ override public void fill(int amount)
// }
}
}
}
}
20 changes: 10 additions & 10 deletions SqlScriptDom/antlr/antlr/CharBuffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ override public void fill(int amount)
//{
syncConsume();
// Fill the buffer sufficiently to hold needed characters
int charsToRead = (amount + markerOffset) - queue.Count;
int charsToRead = (amount + markerOffset) - queue.nbrEntries;
int c;

while (charsToRead > 0)
Expand All @@ -71,16 +71,16 @@ override public void fill(int amount)
for (int i = 0; i < c; i++)
{
// Append the next character
queue.Add(buf[i]);
queue.append(buf[i]);
}
charsToRead -= c;
if (c < BUF_SIZE)
charsToRead -= c;
if (c < BUF_SIZE)
{
while (charsToRead-- > 0)
{
queue.Add(CharScanner.EOF_CHAR);
}
break;
while (charsToRead-- > 0)
{
queue.append(CharScanner.EOF_CHAR);
}
break;
}
}
//}
Expand All @@ -90,4 +90,4 @@ override public void fill(int amount)
//}
}
}
}
}
21 changes: 14 additions & 7 deletions SqlScriptDom/antlr/antlr/CharQueue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ namespace antlr
/*A circular buffer object used by CharBuffer */
internal class CharQueue
{
/*Physical circular buffer of tokens */
protected internal char[] buffer;
/*Physical circular buffer of character values and the EOF sentinel */
protected internal int[] buffer;
/*buffer.length-1 for quick modulos */
private int sizeLessOne;
/*physical index of front token */
Expand Down Expand Up @@ -54,7 +54,7 @@ public CharQueue(int minSize)
/*Add token to end of the queue
* @param tok The token to add
*/
public void append(char tok)
public void append(int tok)
{
if (nbrEntries == buffer.Length)
{
Expand All @@ -67,15 +67,15 @@ public void append(char tok)
/*Fetch a token from the queue by index
* @param idx The index of the token to fetch, where zero is the token at the front of the queue
*/
public char elementAt(int idx)
public int elementAt(int idx)
{
return buffer[(offset + idx) & sizeLessOne];
}

/*Expand the token buffer by doubling its capacity */
private void expand()
{
char[] newBuffer = new char[buffer.Length * 2];
int[] newBuffer = new int[buffer.Length * 2];
// Copy the contents to the new buffer
// Note that this will store the first logical item in the
// first physical array element.
Expand All @@ -95,7 +95,7 @@ private void expand()
public virtual void init(int size)
{
// Allocate buffer
buffer = new char[size];
buffer = new int[size];
// Other initialization
sizeLessOne = size - 1;
offset = 0;
Expand All @@ -116,5 +116,12 @@ public void removeFirst()
offset = (offset + 1) & sizeLessOne;
nbrEntries--;
}

/* Remove chars from the front of the queue */
public void remove(int count)
{
offset = (offset + count) & sizeLessOne;
nbrEntries -= count;
}
}
}
}
52 changes: 26 additions & 26 deletions SqlScriptDom/antlr/antlr/InputBuffer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
namespace antlr
{
using System;
using ArrayList = System.Collections.ArrayList;
using StringBuilder = System.Text.StringBuilder;

/*ANTLR Translator Generator
Expand Down Expand Up @@ -48,12 +47,14 @@ internal abstract class InputBuffer
protected internal int numToConsume = 0;

// Circular queue
protected ArrayList queue;
protected CharQueue queue;

private const int CompactionThreshold = 1024;

/*Create an input buffer */
public InputBuffer()
{
queue = new ArrayList();
queue = new CharQueue(16);
}

/*This method updates the state of the input buffer so that
Expand Down Expand Up @@ -81,23 +82,23 @@ public virtual string getLAChars()
{
StringBuilder la = new StringBuilder();

// copy buffer contents to array before looping thru contents (it's usually faster)
char[] fastBuf = new char[queue.Count-markerOffset];
queue.CopyTo(fastBuf, markerOffset);
for (int i = markerOffset; i < queue.nbrEntries; i++)
{
la.Append((char)queue.elementAt(i));
}

la.Append(fastBuf);
return la.ToString();
}

public virtual string getMarkedChars()
{
StringBuilder marked = new StringBuilder();

// copy buffer contents to array before looping thru contents (it's usually faster)
char[] fastBuf = new char[queue.Count-markerOffset];
queue.CopyTo(fastBuf, markerOffset);
for (int i = markerOffset; i < queue.nbrEntries; i++)
{
marked.Append((char)queue.elementAt(i));
}

marked.Append(fastBuf);
return marked.ToString();
}

Expand All @@ -113,7 +114,7 @@ public virtual int LA(int i)
// Entries are boxed char values for real input and a boxed int sentinel
// (CharScanner.EOF_CHAR) for end-of-input; Convert.ToInt32 handles both so a
// real U+FFFF (65535) stays distinct from the EOF sentinel (65536).
return Convert.ToInt32(queue[markerOffset + i - 1]);
return queue.elementAt(markerOffset + i - 1);
}

/*Return an integer marker that can be used to rewind the buffer to
Expand Down Expand Up @@ -143,26 +144,25 @@ public virtual void reset()
nMarkers = 0;
markerOffset = 0;
numToConsume = 0;
queue.Clear();
queue.reset();
}

/*Sync up deferred consumption */
protected internal virtual void syncConsume()
{
if (numToConsume > 0)
if (numToConsume == 0)
{
return;
}

markerOffset += numToConsume;
numToConsume = 0;

if (nMarkers == 0 && markerOffset >= CompactionThreshold)
{
if (nMarkers > 0)
{
// guess mode -- leave leading characters and bump offset.
markerOffset += numToConsume;
}
else
{
// normal mode -- remove "consumed" characters from buffer
queue.RemoveRange(0, numToConsume);
}
numToConsume = 0;
queue.remove(markerOffset);
markerOffset = 0;
}
}
}
}
}