neovars: compose generator/parser update

- drop ANSI-Autohotkey support, make AutoHotkey_L Unicode mandatory
  (ANSI wasn't fully supported anyways, also compare 41738845d1)
- new tool compose-update.ahk replaces make-compose.bat
- removed makecompose.ahk, which was mostly redundant
  (but subtly different in Unicode- vs. ANSI native encoding)
- new compose-parse.ahk and util.ahk replace most of makecompose.ahk and
  compose-gen.ahk, and improves performance
- configurable compose-update.ahk and dynamic compose (on startup):
  compose-files can be specified in ini-files (fully backwarts-compatible
  with the "dynamischesCompose" Neo2.ini setting)
This commit is contained in:
ferdinym 2020-10-20 03:23:15 +02:00
parent 116cf47c42
commit 0d0691ab35
16 changed files with 2423 additions and 4055 deletions

2
.gitignore vendored
View File

@ -49,7 +49,9 @@
/windows/neo-vars/src/neo20-r*.ahk
/windows/neo-vars/src/_gitwcrev.generated.ahk
/windows/neo-vars/src/compose.generated.ahk~
/windows/neo-vars/src/compose-tainted.generated.ahk
/windows/neo-vars/src/compose-tainted.generated.ahk~
/windows/neo-vars/src/config-iscompiled1.generated.ahk
# /yaml/

View File

@ -0,0 +1,25 @@
; Compsoe-Konfiguration
;
; Diese Datei wird von tools/compose-update.ahk gelesen.
; Durch Ändern der Werte kann die Komposita-Generierung angepasst werden.
; Dabei gilt stets 0 = Nein, 1 = Ja (soweit nicht anders angegeben).
;
; Hinweise:
; - Diese Datei muss in "ANSI"-Kodierung gespeichert werden.
[Global]
; Zeichenkette, welche allen Dateinamen in composeFiles vorangestellt wird.
; Alle notwendigen (Back-)Slashes müssen enthalten sein!
composeFilesPrefix=..\..\..\Compose\src\
; Komma-separierte Liste der zu ladenden compose-Dateien.
; Nicht-absolute Pfade werden relativ zum "tools"-Verzeichnis ausgewertet.
; In einer vollständigen lokalen Kopie des neo-layout Projektes
; können die Dateien unter "..\..\..\Compose\src" verwendet werden.
composeFiles=en_US.UTF-8, base.module, greek.module, math.module, cyrillic.module, lang.module
; Sollen von den bestehenden Skripten mit früheren Compose-Definitionen
; Sicherheitskopien erstellt werden, bevor diese gelöscht werden?
; Hinweis: Sicherheitskopien wird eine Tilde "~" an den Dateinamen angehängt.
makeBackups=1

View File

@ -105,15 +105,25 @@ Mod4LockOff=0
NumLockOff=0
; SONSTIGE EINSTELLUNGEN -------------
; COMPOSE-SEQUENZEN ------------------
; Sollen Compose-Kombinationen dynamisch aus XCompose-Dateien erzeugt werden?
dynamischesCompose=0
; Zeichenkette, welche allen Dateinamen in dynComposeFiles vorangestellt wird.
; Alle notwendigen (Back-)Slashes müssen enthalten sein!
dynComposeFilesPrefix=..\..\..\Compose\src\
; Komma-separierte Liste der bei dynamischem Compose geladenen Dateien.
; Hinweise: Nicht-absolute Pfade werden relativ zum Programmverzeichnis ausgewertet.
; Werden keine Dateien angegeben, werden einige Standarddateien innerhalb
; einer lokalen Kopie des neo-layout Projektverzeichnisses gesucht.
dynComposeFiles=en_US.UTF-8, base.module, greek.module, math.module, cyrillic.module, lang.module, weiter_Definitionen.txt
; WEITERE EINSTELLUNGEN --------------
; Soll das Programm deaktiviert gestartet werden?
; Hinweis: Kann jederzeit über das Tray-Icon oder per Shift + Pause (de)aktiviert werden
startSuspended=0
; Sollen Compose-Kombinationen dynamisch aus einer lokalen Kopie des Repositorys erzeugt werden?
dynamischesCompose=0
; SONSTIGES --------------------------

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -3,13 +3,28 @@
name=Neo 2.0 r%revision%-r%CompRevision% (%A_ScriptName%)
enable=Aktiviere %name%
disable=Deaktiviere %name%
#LTrim ; Quelltext kann eingerückt werden
#LTrim ; Quelltext kann eingerückt werden
FileEncoding, UTF-8 ; Read/write files in utf-8 encoding by default
NEONumLockLEDState := "Off"
NEOCapsLockLEDState := "Off"
NEOScrollLockLEDState := "Off"
OnExit, exitprogram
; native unicode-AHK required
if (not A_IsUnicode) {
MsgBox, 52, Fehlende Unicode-Unterstützung!,
(LTrim
Das Skript benötigt die Unicode-Version von AutoHotkey,
wird aber von ANSI-AutoHotkey ausgeführt.
Trotzdem fortfahren?`n
)
IfMsgBox, No
ExitApp
IfMsgBox, Yes
Suspend
}
if isPortable
ApplicationFolder := A_ScriptDir . "\Neo2-portable"
else
@ -103,19 +118,18 @@ SetOldLockStates() {
}
%EbeneAktualisieren%()
ActivateLayOut(inputlocale)
TheKeys()
ChangeCustomLayout()
Check_BSTUpdate()
; Compose definitions
if (dynamischesCompose)
LoadCurrentCompose()
else
LoadDefaultCompose()
LoadToolCompose() ; may overwrite prior defs
LoadToolCompose()
userCompFun := "LoadUserdefinedCompose"
if (IsFunc(userCompFun))
%userCompFun%()

View File

@ -147,14 +147,14 @@ EDR(pos,caps,e1,e2,e3,e4,e5,e6,e7="",e8="") {
ED(pos,caps,e1a,e2a,e3a,e4a,e5a,e6a,e7a="",e8a="") {
global
e1 := EncodeUniComposeA(e1a)
e2 := EncodeUniComposeA(e2a)
e3 := EncodeUniComposeA(e3a)
e4 := EncodeUniComposeA(e4a)
e5 := EncodeUniComposeA(e5a)
e6 := EncodeUniComposeA(e6a)
e7 := EncodeUniComposeA(e7a)
e8 := EncodeUniComposeA(e8a)
e1 := util_char2UCode(e1a)
e2 := util_char2UCode(e2a)
e3 := util_char2UCode(e3a)
e4 := util_char2UCode(e4a)
e5 := util_char2UCode(e5a)
e6 := util_char2UCode(e6a)
e7 := util_char2UCode(e7a)
e8 := util_char2UCode(e8a)
EDR(pos,caps,e1,e2,e3,e4,e5,e6,e7,e8)
}
@ -195,7 +195,7 @@ EDSKUpdate(la, ea, eb) {
scpos := LOSP%lspos%
pos := vksc%scpos%
; set properties
SetKeyPos("CP" . eb . pos, EncodeUniComposeA(ea))
SetKeyPos("CP" . eb . pos, util_char2UCode(ea))
}
Change1256Layout(newlayoutstring) {
@ -224,7 +224,7 @@ Change1256Layout(newlayoutstring) {
Loop, Parse, newlayoutstring
{
e1a := A_LoopField
e1 := EncodeUniComposeA(e1a)
e1 := util_char2UCode(e1a)
scpos := LOSP%A_Index%
pos := vksc%scpos%
; apply cached properties (plus new level 1 char.) in new order

View File

@ -36,14 +36,14 @@ CharProc__LnS1() {
; ſ, s und ß zyklisch vertauschen auf den drei Positionen
scpos := LOSP%spos%
pos := vksc%scpos%
SetKeyPos("CP1" . pos, EncodeUniComposeA("ſ"))
SetKeyPos("CP1" . pos, util_char2UCode("ſ"))
scpos := LOSP%eszettpos%
pos := vksc%scpos%
SetKeyPos("CP1" . pos, EncodeUniComposeA("s"))
SetKeyPos("CP1" . pos, util_char2UCode("s"))
pos := vksc01A
SetKeyPos("CP3" . pos, EncodeUniComposeA("ß"))
SetKeyPos("CP3" . pos, util_char2UCode("ß"))
}
CharProc__LnS0() {
@ -55,12 +55,12 @@ CharProc__LnS0() {
; ſ, s und ß wieder auf die Originalpositionen setzen
scpos := LOSP%spos%
pos := vksc%scpos%
SetKeyPos("CP1" . pos, EncodeUniComposeA("s"))
SetKeyPos("CP1" . pos, util_char2UCode("s"))
scpos := LOSP%eszettpos%
pos := vksc%scpos%
SetKeyPos("CP1" . pos, EncodeUniComposeA("ß"))
SetKeyPos("CP1" . pos, util_char2UCode("ß"))
pos := vksc01A
SetKeyPos("CP3" . pos, EncodeUniComposeA("ſ"))
SetKeyPos("CP3" . pos, util_char2UCode("ſ"))
}

File diff suppressed because it is too large Load Diff

View File

@ -3,15 +3,18 @@
SetWorkingDir, %A_ScriptDir%
#include %A_ScriptDir%\
#include util.ahk
; Konfiguration des interpretierten/kompilierten Skriptes
; Lädt optional benutzerdefinierte Skripte (custom.ahk)
#include *i config-iscompiled%A_IsCompiled%.ahk
#include *i config-iscompiled%A_IsCompiled%.generated.ahk
; die Compose-Definitionen
#include compose-parse.ahk
#include compose-gen.ahk
#include *i compose.generated.ahk
#include *i compose-tainted.generated.ahk
#include compose-gen.ahk
; Hier liegt die Tastaturbelegung
#include keydefinitions.ahk

View File

@ -894,7 +894,7 @@ CharProc__BSTK() {
; Set style="t" to highlight chars as a tool (typically starts on "P")
GUISYM(sym,chars,style:=" ") {
global
GSYM%sym% := style . EncodeUniCompose(chars)
GSYM%sym% := style . util_str2UCodes(chars)
}
BSTSymbols() {

View File

@ -59,7 +59,7 @@ CharProc__Rom4() {
GenRomanDigit(Pos, DigitIs, DigitTest, str0, str1, str2, str3, str4, str5) {
res := ""
if (DigitIs == DigitTest)
res := EncodeUniComposeA(str%Pos%)
res := util_char2UCode(str%Pos%)
return res
}
@ -160,7 +160,7 @@ PressHookRoman(PhysKey, ActKey, Char) {
RomanPos := RomanPos + 1
}
loop {
if (RomanStr == "")
if (RomanStr == "")
break ; erledigt
CharOut(SubStr(RomanStr,1,7))
RomanStr := SubStr(RomanStr,8)
@ -262,57 +262,6 @@ CharOutFilterDUni(char,down,up) {
return char
}
EncodeUni(str) {
SetFormat, Integer, hex
; MsgBox % Asc(SubStr(str,1,1)) . Asc(SubStr(str,2,1))
result := ""
loop {
char := asc(SubStr(str,1,1))
str := SubStr(str,2)
if (char < 0x80)
result .= "U" . SubStr("000000" . SubStr(char,3),-5)
else if (char < 0xC0) {
; error
} else if (char < 0xE0) {
char2 := asc(Substr(str,1,1))
str := SubStr(str,2)
if ((char2 < 0x80) or (char2 > 0xBF)) {
; error
} else {
result .= "U" . SubStr("000000" . SubStr((((char & 0x1F) << 6) + (char2 & 0x3F)),3),-5)
}
} else if (char < 0xF8) {
char2 := asc(SubStr(str,1,1))
char3 := asc(SubStr(str,2,1))
str := SubStr(str,3)
if ((char2 < 0x80) or (char2 > 0xBF)
or (char3 < 0x80) or (char3 > 0xBF)) {
; error
} else {
result .= "U" . SubStr("000000" . SubStr((((char & 0x0F) << 12) + ((char2 & 0x3F) << 6) + (char3 & 0x3F)),3),-5)
}
} else if (char < 0xFC) {
char2 := asc(SubStr(str,1,1))
char3 := asc(SubStr(str,2,1))
char3 := asc(SubStr(str,3,1))
str := SubStr(str,4)
if ( (char2 < 0x80) or (char2 > 0xBF)
or (char3 < 0x80) or (char3 > 0xBF)
or (char4 < 0x80) or (char4 > 0xBF)) {
; error
} else {
result .= "U" . SubStr("000000" . SubStr((((char & 0x07) << 18) + ((char2 & 0x3F) << 12) + ((char3 & 0x3F) << 6) + (char4 & 0x3F)),3),-5)
}
}
if (str == "")
break
}
SetFormat, Integer, d
StringUpper,result,result
return result
}
; Simple calculator
LoadToolComposeCalc() {
global
@ -476,7 +425,7 @@ PressHookCalc(PhysKey, ActKey, Char) {
CalcResult := CalcResult + 0
SetFormat,Integer,d
}
tosend := EncodeUni(CalcResult)
tosend := util_char2UCode(CalcResult)
if (CalcEcho) {
Char := "U00003D"
PP%PhysKey% := Char
@ -491,7 +440,7 @@ PressHookCalc(PhysKey, ActKey, Char) {
CharOut(SubStr(tosend,1,7))
}
tosend := SubStr(tosend,8)
if (tosend == "")
if (tosend == "")
break ; erledigt
}
PressHookProc := ""

View File

@ -0,0 +1,77 @@
; -*- encoding: utf-8 -*-
; Noovars utilities module
; Transform string into sequence of codepoints "UxxxxxxUyyyyyyU..."
;
; Input string must be UTF-16 encoded (native in Unicode-AutoHotkey).
; For each unicode character encountered in the input string,
; it's unicode-codepoint is appended to the output string in the
; format Uxxxxxx where U is a leading "U" character, followed by
; the (right-adjusted, zero-padded, six-digit upper-case) hexadecimal
; representation of the codepoint.
;
; Example: util_str2UCodes("A♫7") returns "U000041U00266BU000037"
;
util_str2UCodes(str) {
result := ""
i := 1
While (i <= StrLen(str)) {
cp := Ord(SubStr(str,i,2))
i += 1 + (cp >= 0x10000)
result .= Format("U{:06X}", cp)
}
return result
}
; Transform unicode-character into codepoint string "Uxxxxxx" or keyId
;
; If input is a zero- or 7-character sequence, it is returned as is.
; (It is assumed that a 7 character sequence is a neovars keyId, i.e. either
; a "Uxxxxxx" format or a 7-character special key name such as "S__Comp").
;
; Otherwise input shall be either a single utf-16 encoded unicode character
; (StrLen of 1 or 2), and it's unicode codepoint is returned in the
; string "Uxxxxxx" where xxxxxx represent the right-adjusted, zero-padded
; six-digit upper-case hexadecimal representation of the codepoint.
;
; Examples:
; util_char2UCode("") returns ""
; util_char2UCode("A") returns "U000041"
; util_char2UCode("U000041") returns "U000041"
; util_char2UCode("T__tlde") returns "T__tlde"
;
util_char2UCode(str) {
if (str == "" or StrLen(str) == 7)
return str
return Format("U{:06X}", Ord(str))
}
; Unescapes string by performing the following substitutions:
; - Replace "\\" by "\"
; - Replace "\n" by line-feed
; - Replace "\x" by "x" (where x is any character other than "\", "n")
; - Remove a trailing "\"
; Replacements are performed left-to-right and do not overlap.
; An alternative (single) escape character can be provided with esc.
util_strUnescape(str, esc:="\") {
static LF_CHAR := chr(0x000d) ; line-feed
out := ""
, j := 1 ; pos. of next character added to output
, k := 1 ; pos. of next character to be scanned
Loop {
; scan for next "\"
k := InStr(str, "\", true, k)
if not k
break
out .= SubStr(str, j, k - j)
, j := k + 1
, k += 2
; substitions
if (SubStr(str, j, 1) == "n") {
out .= LF_CHAR
, j++
}
}
return out . SubStr(str, j)
}

View File

@ -0,0 +1,207 @@
; Generate script with compose-definitions from XCompose files
;
; Usage:
; Run as standalone script with AutoHotkey.exe.
;
; Generates the AHK-source-script which sets up the available compose-
; sequences in the neovars driver.
;
; Compose-sequences must be provided in input files, whose names along
; with other configuration are read from "..\config\compose-config.ini",
;
; The output script is written to "..\src\compos.generated.ahk", if all
; input files reside inside the neovars git-repository and have no uncommitted
; changes; or "..\src\composee-tainted.generated.ahk" otherwise.
; If either output script exist, it will be removed. Depending on
; configuration options, a backup copy is made (overwriting existing backups).
;
; Only a subset of the XCompose syntax is supported. For instance,
; input files shall not define includes, and must not specify modifiers.
; See also https://www.x.org/releases/X11R7.7/doc/man/man5/Compose.5.xhtml
FileEncoding, UTF-8
logInit("Compose-update")
SetWorkingDir %A_ScriptDir%
#Include %A_ScriptDir%\
#Include script/logwindow.ahk
#Include script/util.ahk
#include ../src/compose-parse.ahk
#include ../src/util.ahk
; Paths
srcDir := A_ScriptDir . "\..\src"
binDir := A_ScriptDir . "\..\bin"
configDir := A_ScriptDir . "\..\config"
iniFile := util_getFullPath(configDir . "\compose-config.ini")
;
; Read user-defined configuration
;
; Default configs (keys are not case-sensitive)
userConfigDefaults := {composeFilesPrefix: "", composeFiles: "", makeBackups: True}
configs := {revision: "<unknown>", composeFilesFull: [], outputFile: ""}
; Read user-defined configs from ini-file
if not FileExist(iniFile)
{
logError("Configuration file not found at '" . iniFile . "'")
logFinal()
}
logEntry("Read user-defined configuration...")
for key, value in userConfigDefaults
{
IniRead, valueNew, %iniFile%, Global, %key% , %value%
configs[key] := valueNew
}
; Asseble full pathnames for input files
composeFilesCSV := configs["composeFiles"]
Loop, parse, composeFilesCSV, CSV, %A_Space%%A_Tab%
{
configs["composeFilesFull"].push(util_getFullPath(configs["composeFilesPrefix"] . A_LoopField))
}
numComposeFiles := configs["composeFilesFull"].Length()
if (numComposeFiles == 0)
logWarning("No compose-files provided! Compose-definitions will be empty.")
; Validate input filenames (early exit on errors)
Loop, % numComposeFiles
{
file := configs["composeFilesFull"][A_Index]
if not FileExist(file)
{
logError("Compose-file not found at '" . file . "'")
logFinal()
}
}
;
; Other configurations
;
; Query revsion information for the compose files (requires git)
; If no revision can be determined (e.g. git not installed or input-
; files from outside the repository), a warning will be printed.
logEntry("Query revision information...")
composePathsQuoted := ""
for i, file in configs["composeFilesFull"]
composePathsQuoted .= """" . file . """ "
exitCode := util_runWaitCMD("git rev-list -n 1 HEAD -- " . composePathsQuoted, strOut, strErr)
if not exitCode and StrLen(strOut) >= 7
configs["revision"] := SubStr(strOut, 1, 7)
else
logWarning("Could not query revision information from git repository. " . strErr)
; Clear any existing output files (if exist)
logEntry("Deleting old compose scripts")
outputFileDefault := util_getFullPath(srcDir . "\compose.generated.ahk")
outputFileTainted := util_getFullPath(srcDir . "\compose-tainted.generated.ahk")
for i,f in [outputFileDefault, outputFileTainted]
{
; Move existing scripts to backup copy (overwrite existing backups)
if (configs["makeBackups"])
{
backupFile := f . "~"
FileMove, % f, % backupFile, 1
}
else
FileDelete, % f
if FileExist(f)
{
logError("Could not remove old compose script '" . f . "'")
logFinal()
}
}
; Name of the output script (in source-directory):
; - If uncommitted changes pending: (over-)write to "compose-tainted.generated.ahk"
; - If working-dir clean: (over-)write to "compose.generated.ahk"
exitCode := util_runWaitCMD("git diff --quiet -- " . composePathsQuoted)
configs["outputFile"] := (exitCode ? outputFileTainted : outputFileDefault)
logEntry("Set output file to '" . configs["outputFile"] . "'")
;
; Parse compose-definitions into source script
;
; Write header
logEntry("Write header information.")
scriptGeneratedBy := A_ScriptName
scriptLineRevision := "compRevision := """ . configs["revision"] . """"
FileAppend,
(
; -*- encoding: utf-8 -*-
;
; ** THIS FILE WAS GENERATED BY %scriptGeneratedBy% **
; ** DO NOT EDIT BY HAND -- FILE MAY BE OVERWRITTEN ANYTIME **
;
; Revision information
%scriptLineRevision%
; Make compose-definitions globally available
LoadDefaultCompose() {
global
), % configs["outputFile"], UTF-8
; Write compose-definitions
onParseErrorFun := Func("onParseError")
onProgressFun := Func("onProgress")
keySym2KeyIdMap := makekeySym2KeyIdMap()
totalErrorCount := 0
Loop, % numComposeFiles
{
file := configs["composeFilesFull"][A_Index]
logEntry("Parse compose-definitions from '" . file . "'...")
if not FileExist(file)
{
logError("Compose-file not found at '" . file . "'")
continue ; try to complete output with footer etc.
}
try
totalErrorCount += parseComposeFile(file, keySym2KeyIdMap, configs["outputFile"], onParseErrorFun, onProgressFun)
catch e
{
logError("Unhandled exception '" . e.What . (e.Message == "" ? "" : "' reports '" . e.Message) . "' while processing '" . file . "'")
continue ; try to complete output with footer etc.
}
}
logEntry(Format("Parsing finished with {:i} parsing-error{:s} in {:i} file{:s}.", totalErrorCount, util_pluralS(totalErrorCount), numComposeFiles, util_pluralS(numComposeFiles)))
; Write footer
logEntry("Write footer information.")
FileAppend, }, % configs["outputFile"], UTF-8
; Exit
logEntry("Compose-update completed.")
logFinal()
; -------------------------------------
; Functions
; -------------------------------------
; Compose-Parser progress handler
onParseError(e, errorCount)
{
if (e.type == "key_sym")
logWarning(Format("Ignored unknown <{:s}> on line {:u} '{:s}'", e.keySym, e.lineNo, e.line))
else if (e.type == "result")
logWarning(Format("Ignored empty result on line {:u} '{:s}'", e.lineNo, e.line))
else
logWarning(Format("Ignored bad line {:u} '{:s}'", e.lineNo, e.line))
}
; Compose-Parser error handler
onProgress(count, total, errorCount)
{
progress := 100 * count / total
msgStr := Format("complete - {:u} line{:s} with {:u} parsing-error{:s}", count, util_pluralS(count), errorCount, util_pluralS(errorCount))
logProgressUpdate(progress, msgStr)
if (count == total)
logProgressStop()
}

View File

@ -1,30 +0,0 @@
@echo off
echo Setting default local path variables
set ahkpath=C:\Programme\AutoHotkey
set AutoHotKey=%ahkpath%\AutoHotKeyA32.exe
set srcdir=..\src
set bindir=..\bin
REM The path to the directory used for generating a consistent SVN version (revision number)
set gitversiondir=..\..\..\Compose
echo Getting git revision
for /f "tokens=* USEBACKQ" %%R in (`"git rev-list -n 1 HEAD -- %gitversiondir%"`) do set CompRevision=%%R
set CompRevision=%CompRevision:~0,7%
set fncomp=%srcdir%\compose.generated.ahk
git diff --quiet -- "%gitversiondir%" > nul
if %ERRORLEVEL% EQU 1 (
set fncomp=%srcdir%\compose-tainted.generated.ahk
)
echo Deleting old compose sequences
del "%srcdir%\Compose.generated.ahk" "%srcdir%\Compose-tainted.generated.ahk" 2> nul
echo Compiling compose sequences
"%AutoHotkey%" "%srcdir%\makecompose.ahk" "%CompRevision%" "%fncomp%" "%gitversiondir%\src\en_US.UTF-8" "%gitversiondir%\src\base.module" "%gitversiondir%\src\greek.module" "%gitversiondir%\src\math.module" "%gitversiondir%\src\lang.module" "%gitversiondir%\src\weiter_Definitionen.txt"
echo Compose update complete! You can now close this log window.
pause

View File

@ -47,3 +47,9 @@ util_getFullPath(path)
VarSetCapacity(buf, cc * (A_IsUnicode ? 2 : 1))
return DllCall("GetFullPathName", "str", path, "uint", cc, "str", buf, "ptr", 0) ? buf : ""
}
; Return "s" if val is plural or zero, "" otherwise
util_pluralS(val)
{
return (Abs(val) == 1 ? "" : "s")
}