diff --git a/LLMFarm/AIChatModel.swift b/LLMFarm/AIChatModel.swift index 05587fb..6afd4c7 100644 --- a/LLMFarm/AIChatModel.swift +++ b/LLMFarm/AIChatModel.swift @@ -102,6 +102,10 @@ final class AIChatModel: ObservableObject { } do{ if chat_config!["model_inference"] as! String == "llama"{ + if (chat_config!["grammar"] != nil && chat_config!["grammar"] as! String != "" && chat_config!["grammar"] as! String != ""){ + let grammar_path = get_grammar_path_by_name(chat_config!["grammar"] as! String) + model_context_param.grammar_path = grammar_path + } if modelURL.hasSuffix(".gguf"){ try model_load_res = self.chat?.loadModel(ModelInference.LLama_gguf,contextParams: model_context_param) }else{ diff --git a/LLMFarm/Lib/FileHelper.swift b/LLMFarm/Lib/FileHelper.swift index 54d11eb..830c4fc 100644 --- a/LLMFarm/Lib/FileHelper.swift +++ b/LLMFarm/Lib/FileHelper.swift @@ -239,6 +239,53 @@ public func get_models_list() -> [Dictionary]?{ return res } +public func get_grammar_path_by_name(_ grammar_name:String) -> String?{ + do { + let fileManager = FileManager.default + let documentsPath = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first + let destinationURL = documentsPath!.appendingPathComponent("grammars") + try fileManager.createDirectory (at: destinationURL, withIntermediateDirectories: true, attributes: nil) + let path = destinationURL.appendingPathComponent(grammar_name).path + if fileManager.fileExists(atPath: path){ + return path + }else{ + return nil + } + + } catch { + print(error) + } + return nil +} + +public func get_grammars_list() -> [String]?{ + var res: [String] = [] + res.append("") + do { +// var gbnf_path=Bundle.main.resourcePath!.appending("/grammars") +// let gbnf_files = try FileManager.default.contentsOfDirectory(atPath: gbnf_path) +// for gbnf_file in gbnf_files { +// let tmp_chat_info = ["file_name":gbnf_file,"location":"res"] +// res.append(tmp_chat_info) +// } + let fileManager = FileManager.default + let documentsPath = fileManager.urls(for: .documentDirectory, in: .userDomainMask).first + let destinationURL = documentsPath!.appendingPathComponent("grammars") + try fileManager.createDirectory (at: destinationURL, withIntermediateDirectories: true, attributes: nil) + let files = try fileManager.contentsOfDirectory(atPath: destinationURL.path) + for gbnf_file in files { + if gbnf_file.hasSuffix(".gbnf"){ +// let tmp_chat_info = ["file_name":gbnf_file,"location":"doc"] + res.append(gbnf_file) + } + } + return res + } catch { + // failed to read directory – bad permissions, perhaps? + } + return res +} + //func get_config_by_model_name(_ model_name:String) -> Dictionary?{ // do { //// let index = model_name.index(model_name.startIndex, offsetBy:model_name.count-4) diff --git a/LLMFarm/Settings/AddChatView.swift b/LLMFarm/Settings/AddChatView.swift index 70d4f93..0f50de4 100644 --- a/LLMFarm/Settings/AddChatView.swift +++ b/LLMFarm/Settings/AddChatView.swift @@ -85,6 +85,7 @@ struct AddChatView: View { @State private var isImporting: Bool = false @State private var tfs_z: Float = 1.0 @State private var typical_p: Float = 1.0 + @State private var grammar: String = "" var hardware_arch = Get_Machine_Hardware_Name() @Binding var renew_chat_list: () -> Void @@ -106,6 +107,9 @@ struct AddChatView: View { @State var models_previews = get_models_list()! + @State var grammars_previews = get_grammars_list()! + + init(add_chat_dialog: Binding,edit_chat_dialog:Binding, renew_chat_list: Binding<() -> Void>) { self._add_chat_dialog = add_chat_dialog @@ -201,6 +205,9 @@ struct AddChatView: View { if (chat_config!["typical_p"] != nil){ self._typical_p = State(initialValue: chat_config!["typical_p"] as! Float) } + if (chat_config!["grammar"] != nil){ + self._grammar = State(initialValue: chat_config!["grammar"]! as! String) + } } func apply_setting_template(template:ModelSettingsTemplate){ @@ -277,7 +284,8 @@ struct AddChatView: View { "mirostat_eta":mirostat_eta, "mirostat_tau":mirostat_tau, "tfs_z":tfs_z, - "typical_p":typical_p + "typical_p":typical_p, + "grammar":grammar ] _ = create_chat(options,edit_chat_dialog:self.edit_chat_dialog,chat_name:self.chat_name) if add_chat_dialog { @@ -412,6 +420,21 @@ struct AddChatView: View { .padding(.horizontal) .padding(.top, 8) + if model_inference == "llama"{ + HStack{ + Text("Grammar sampling:") + .frame(maxWidth: .infinity, alignment: .leading) + Picker("", selection: $grammar) { + ForEach(grammars_previews, id: \.self) { + Text($0) + } + } + .pickerStyle(.menu) + + } + .padding(.horizontal) + .padding(.top, 8) + } DisclosureGroup("Prompt format:", isExpanded: $isPromptAccordionExpanded) { Group { @@ -531,6 +554,7 @@ struct AddChatView: View { DisclosureGroup("Sampling options:", isExpanded: $isSamplingAccordionExpanded) { Group { + HStack{ Text("Sampling:") .frame(maxWidth: .infinity, alignment: .leading) @@ -559,7 +583,7 @@ struct AddChatView: View { } .padding(.horizontal) .padding(.top, 8) - + if model_sampling == "temperature" { Group { diff --git a/LLMFarm/grammars/arithmetic.gbnf b/LLMFarm/grammars/arithmetic.gbnf new file mode 100644 index 0000000..3aa95a9 --- /dev/null +++ b/LLMFarm/grammars/arithmetic.gbnf @@ -0,0 +1,6 @@ +root ::= (expr "=" ws term "\n")+ +expr ::= term ([-+*/] term)* +term ::= ident | num | "(" ws expr ")" ws +ident ::= [a-z] [a-z0-9_]* ws +num ::= [0-9]+ ws +ws ::= [ \t\n]* diff --git a/LLMFarm/grammars/c.gbnf b/LLMFarm/grammars/c.gbnf new file mode 100644 index 0000000..4a0331d --- /dev/null +++ b/LLMFarm/grammars/c.gbnf @@ -0,0 +1,42 @@ +root ::= (declaration)* + +declaration ::= dataType identifier "(" parameter? ")" "{" statement* "}" + +dataType ::= "int" ws | "float" ws | "char" ws +identifier ::= [a-zA-Z_] [a-zA-Z_0-9]* + +parameter ::= dataType identifier + +statement ::= + ( dataType identifier ws "=" ws expression ";" ) | + ( identifier ws "=" ws expression ";" ) | + ( identifier ws "(" argList? ")" ";" ) | + ( "return" ws expression ";" ) | + ( "while" "(" condition ")" "{" statement* "}" ) | + ( "for" "(" forInit ";" ws condition ";" ws forUpdate ")" "{" statement* "}" ) | + ( "if" "(" condition ")" "{" statement* "}" ("else" "{" statement* "}")? ) | + ( singleLineComment ) | + ( multiLineComment ) + +forInit ::= dataType identifier ws "=" ws expression | identifier ws "=" ws expression +forUpdate ::= identifier ws "=" ws expression + +condition ::= expression relationOperator expression +relationOperator ::= ("<=" | "<" | "==" | "!=" | ">=" | ">") + +expression ::= term (("+" | "-") term)* +term ::= factor(("*" | "/") factor)* + +factor ::= identifier | number | unaryTerm | funcCall | parenExpression +unaryTerm ::= "-" factor +funcCall ::= identifier "(" argList? ")" +parenExpression ::= "(" ws expression ws ")" + +argList ::= expression ("," ws expression)* + +number ::= [0-9]+ + +singleLineComment ::= "//" [^\n]* "\n" +multiLineComment ::= "/*" ( [^*] | ("*" [^/]) )* "*/" + +ws ::= ([ \t\n]+) diff --git a/LLMFarm/grammars/chess.gbnf b/LLMFarm/grammars/chess.gbnf new file mode 100644 index 0000000..ef0fc1b --- /dev/null +++ b/LLMFarm/grammars/chess.gbnf @@ -0,0 +1,13 @@ +# Specifies chess moves as a list in algebraic notation, using PGN conventions + +# Force first move to "1. ", then any 1-2 digit number after, relying on model to follow the pattern +root ::= "1. " move " " move "\n" ([1-9] [0-9]? ". " move " " move "\n")+ +move ::= (pawn | nonpawn | castle) [+#]? + +# piece type, optional file/rank, optional capture, dest file & rank +nonpawn ::= [NBKQR] [a-h]? [1-8]? "x"? [a-h] [1-8] + +# optional file & capture, dest file & rank, optional promotion +pawn ::= ([a-h] "x")? [a-h] [1-8] ("=" [NBKQR])? + +castle ::= "O-O" "-O"? diff --git a/LLMFarm/grammars/japanese.gbnf b/LLMFarm/grammars/japanese.gbnf new file mode 100644 index 0000000..43f25ab --- /dev/null +++ b/LLMFarm/grammars/japanese.gbnf @@ -0,0 +1,7 @@ +# A probably incorrect grammar for Japanese +root ::= jp-char+ ([ \t\n] jp-char+)* +jp-char ::= hiragana | katakana | punctuation | cjk +hiragana ::= [ぁ-ゟ] +katakana ::= [ァ-ヿ] +punctuation ::= [、-〾] +cjk ::= [一-鿿] diff --git a/LLMFarm/grammars/json.gbnf b/LLMFarm/grammars/json.gbnf new file mode 100644 index 0000000..a9537cd --- /dev/null +++ b/LLMFarm/grammars/json.gbnf @@ -0,0 +1,25 @@ +root ::= object +value ::= object | array | string | number | ("true" | "false" | "null") ws + +object ::= + "{" ws ( + string ":" ws value + ("," ws string ":" ws value)* + )? "}" ws + +array ::= + "[" ws ( + value + ("," ws value)* + )? "]" ws + +string ::= + "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes + )* "\"" ws + +number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws + +# Optional space: by convention, applied in this grammar after literal chars when allowed +ws ::= ([ \t\n] ws)? diff --git a/LLMFarm/grammars/json_arr.gbnf b/LLMFarm/grammars/json_arr.gbnf new file mode 100644 index 0000000..ef53e77 --- /dev/null +++ b/LLMFarm/grammars/json_arr.gbnf @@ -0,0 +1,34 @@ +# This is the same as json.gbnf but we restrict whitespaces at the end of the root array +# Useful for generating JSON arrays + +root ::= arr +value ::= object | array | string | number | ("true" | "false" | "null") ws + +arr ::= + "[\n" ws ( + value + (",\n" ws value)* + )? "]" + +object ::= + "{" ws ( + string ":" ws value + ("," ws string ":" ws value)* + )? "}" ws + +array ::= + "[" ws ( + value + ("," ws value)* + )? "]" ws + +string ::= + "\"" ( + [^"\\] | + "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes + )* "\"" ws + +number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws + +# Optional space: by convention, applied in this grammar after literal chars when allowed +ws ::= ([ \t\n] ws)? diff --git a/LLMFarm/grammars/list.gbnf b/LLMFarm/grammars/list.gbnf new file mode 100644 index 0000000..51e6c9c --- /dev/null +++ b/LLMFarm/grammars/list.gbnf @@ -0,0 +1,4 @@ +root ::= item+ + +# Excludes various line break characters +item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n" diff --git a/ModelTest/main.swift b/ModelTest/main.swift index 685e3d1..fb77874 100644 --- a/ModelTest/main.swift +++ b/ModelTest/main.swift @@ -43,8 +43,8 @@ func main(){ // modelInference = ModelInference.GPTNeox //// // - // ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/rp-incite-base-v1-3b-ggmlv3-q5_1.bin" - // modelInference = ModelInference.GPTNeox +// ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/rp-incite-base-v1-3b-ggmlv3-q5_1.bin" +// modelInference = ModelInference.GPTNeox // // ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/magicprompt-stable-diffusion-q5_1.bin" // ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/cerebras-2.7b-ggjtv3-q4_0.bin" @@ -57,21 +57,24 @@ func main(){ // modelInference = ModelInference.Starcoder // input_text = "def qsort" // - // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/q4_1-RWKV-4-Raven-1B5-v12-Eng.bin" - // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/RWKV-4-MIDI-120M-v1-20230714-ctx4096-FP16.bin" - // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/Sources/rwkv.cpp-master-8db73b1/tests/tiny-rwkv-660K-FP16.bin" - // modelInference = ModelInference.RWKV - // input_text = "song about love" + ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/q4_1-RWKV-4-Raven-1B5-v12-Eng.bin" +// // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/RWKV-4-MIDI-120M-v1-20230714-ctx4096-FP16.bin" +// // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/Sources/rwkv.cpp-master-8db73b1/tests/tiny-rwkv-660K-FP16.bin" + modelInference = ModelInference.RWKV +// input_text = "song about love" // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/orca-mini-3b.ggmlv3.q4_1.bin" // ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/llama-2-7b-chat-q4_K_M.gguf" // ai.modelPath = "/Users/guinmoon/dev/alpaca_llama_etc/openllama-3b-v2-q8_0.gguf" - ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/orca-mini-3b-q4_1.gguf" - modelInference = ModelInference.LLama_gguf - - +// ai.modelPath = "/Users/guinmoon/Library/Containers/com.guinmoon.LLMFarm/Data/Documents/models/orca-mini-3b-q4_1.gguf" +// modelInference = ModelInference.LLama_gguf +// var params:ModelContextParams = .default - params.use_metal = true +// +// params.use_metal = true + +// params.grammar_path = "/Users/guinmoon/dev/alpaca_llama_etc/LLMFarm/LLMFarm/grammars/list.gbnf" + input_text = "write to do list" do{ try ai.loadModel(modelInference,contextParams: params) @@ -80,28 +83,8 @@ func main(){ return } - //// try? set_promt_format(ai: &ai) - // let exception = tryBlock { - // - //// try? ai.model.promptFormat = .LLaMa - // - // } - // - // if exception != nil { - // print(exception) - // exit(1) - // } - // - // - - // ai.model.promptFormat = .Custom - // ai.model.custom_prompt_format = "Below is an instruction that describes a task. Write a response that appropriately completes the request.### Instruction:{{prompt}}### Response:" - //// - // ai.model.contextParams.seed = 0; - // ai.model.promptStyle = .StableLM_Tuned - // if (!params.lora_adapter.empty()) { // int err = llama_model_apply_lora_from_file(model, @@ -126,20 +109,20 @@ func main(){ //### Response: //""" // - input_text = """ -### User: -Tell more - -### Response: -""" +// input_text = """ +//### User: +//Tell more +// +//### Response: +//""" // var tokens: [llama_token] = [Int32](repeating: 0, count: 256) // var tokens_count:Int = 1 - llama_load_state(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state_.bin") +// llama_load_state(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state_.bin") // llama_load_session_file(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state.bin",tokens.mutPtr, 256,&tokens_count) - let prompt = input_text - let output = try? ai.model.predict(prompt, mainCallback) + + let output = try? ai.model.predict(input_text, mainCallback) // llama_save_session_file(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state.bin",ai.model.session_tokens, ai.model.session_tokens.count) - llama_save_state(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state_.bin") +// llama_save_state(ai.model.context,"/Users/guinmoon/dev/alpaca_llama_etc/dump_state_.bin") // print(output!) } diff --git a/llmfarm_core.swift b/llmfarm_core.swift index e1f2773..f68ba38 160000 --- a/llmfarm_core.swift +++ b/llmfarm_core.swift @@ -1 +1 @@ -Subproject commit e1f2773a9a8c0f8a5fe85c7304a8814e89329800 +Subproject commit f68ba3815bb785904d6fcd1b0f5f32e9df1c3f13