mozhi/utils/engines.go

384 lines
10 KiB
Go
Raw Normal View History

2023-06-25 00:10:57 +05:30
package utils
import (
2023-08-15 14:56:57 +05:30
"errors"
2023-08-30 14:41:20 +05:30
"io"
"net/http"
2023-08-27 19:54:30 +05:30
"os"
2023-08-30 14:41:20 +05:30
"regexp"
2023-08-27 19:54:30 +05:30
"strings"
2023-08-15 14:56:57 +05:30
"github.com/OwO-Network/gdeeplx"
2023-06-25 00:10:57 +05:30
"github.com/gocolly/colly"
"github.com/google/go-querystring/query"
2023-08-15 14:56:57 +05:30
"github.com/google/uuid"
2023-06-25 00:10:57 +05:30
)
2023-06-25 00:11:11 +05:30
2023-08-30 14:41:20 +05:30
var ddgVqd string
func TranslateGoogle(to string, from string, text string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
ToOrig := to
FromOrig := from
// For some reason google uses no for norwegian instead of nb like the rest of the translators. This is for the All function primarily
if to == "nb" {
to = "no"
} else if from == "nb" {
to = "no"
}
var ToValid bool
var FromValid bool
for _, v := range LangListGoogle("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target Language Code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
UserAgent, ok := os.LookupEnv("MOZHI_USER_AGENT")
2023-06-25 00:10:57 +05:30
if !ok {
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
sc := colly.NewCollector(colly.AllowedDomains("translate.google.com"), colly.UserAgent(UserAgent))
var answer string
sc.OnHTML("div.result-container", func(e *colly.HTMLElement) {
answer = e.Text
})
type Options struct {
2023-07-22 19:54:15 +05:30
To string `url:"tl"`
UI string `url:"hl"`
From string `url:"sl"`
Text string `url:"q"`
}
2023-07-22 19:54:15 +05:30
opt := Options{to, to, from, text}
v, _ := query.Values(opt)
2023-07-22 19:54:15 +05:30
url := "https://translate.google.com/m?" + v.Encode()
2023-06-25 00:10:57 +05:30
sc.Visit(url)
var langout LangOut
langout.OutputText = answer
2023-08-16 11:58:03 +05:30
langout.Engine = "google"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
2023-06-25 00:10:57 +05:30
}
2023-08-27 19:54:30 +05:30
func TranslateReverso(to string, from string, query string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
ToOrig := to
FromOrig := from
var ToValid bool
var FromValid bool
for _, v := range LangListReverso("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
2023-06-25 00:11:11 +05:30
json := []byte(`{ "format": "text", "from": "` + from + `", "to": "` + to + `", "input":"` + query + `", "options": {"sentenceSplitter": false, "origin":"translation.web", contextResults: false, languageDetection: true} }`)
reversoOut := PostRequest("https://api.reverso.net/translate/v1/translation", json)
2023-06-25 00:10:57 +05:30
gjsonArr := reversoOut.Get("translation").Array()
var langout LangOut
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "reverso"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
2023-06-25 00:10:57 +05:30
}
2023-08-27 19:54:30 +05:30
func TranslateLibreTranslate(to string, from string, query string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
ToOrig := to
FromOrig := from
var ToValid bool
var FromValid bool
for _, v := range LangListLibreTranslate("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
2023-07-22 19:54:15 +05:30
json := []byte(`{"q":"` + query + `","source":"` + from + `","target":"` + to + `"}`)
2023-07-03 17:29:46 +05:30
// TODO: Make it configurable
libreTranslateOut := PostRequest(os.Getenv("MOZHI_LIBRETRANSLATE_URL")+"/translate", json)
2023-07-03 17:29:46 +05:30
gjsonArr := libreTranslateOut.Get("translatedText").Array()
var langout LangOut
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "libretranslate"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
if from == "auto" {
langout.AutoDetect, _ = AutoDetectLibreTranslate(query)
}
return langout, nil
2023-07-03 17:29:46 +05:30
}
2023-08-27 19:54:30 +05:30
func TranslateWatson(to string, from string, query string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
FromOrig := from
ToOrig := to
var ToValid bool
var FromValid bool
for _, v := range LangListWatson("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
var langout LangOut
if from == "auto" {
langout.AutoDetect, _ = AutoDetectWatson(query)
from = langout.AutoDetect
}
2023-07-22 19:54:15 +05:30
json := []byte(`{"text":"` + query + `","source":"` + from + `","target":"` + to + `"}`)
watsonOut := PostRequest("https://www.ibm.com/demos/live/watson-language-translator/api/translate/text", json)
2023-07-22 11:12:26 +05:30
gjsonArr := watsonOut.Get("payload.translations.0.translation").Array()
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "watson"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
2023-07-22 11:12:26 +05:30
}
2023-08-27 19:54:30 +05:30
func TranslateMyMemory(to string, from string, text string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
FromOrig := from
ToOrig := to
2023-07-23 11:32:48 +05:30
var ToValid bool
var FromValid bool
for _, v := range LangListMyMemory("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
2023-07-23 11:32:48 +05:30
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
2023-07-23 11:32:48 +05:30
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
2023-07-23 11:32:48 +05:30
}
type Options struct {
Translate string `url:"langpair"`
Text string `url:"q"`
}
2023-08-15 14:56:57 +05:30
opt := Options{from + "|" + to, text}
2023-07-23 11:32:48 +05:30
v, _ := query.Values(opt)
myMemoryOut := GetRequest("https://api.mymemory.translated.net/get?" + v.Encode())
gjsonArr := myMemoryOut.Get("responseData.translatedText").Array()
var langout LangOut
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "mymemory"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
2023-07-23 11:32:48 +05:30
}
2023-08-27 19:54:30 +05:30
func TranslateYandex(to string, from string, text string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
FromOrig := from
ToOrig := to
var ToValid bool
var FromValid bool
for _, v := range LangListYandex("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
type Options struct {
2023-07-22 19:54:15 +05:30
Translate string `url:"lang"`
Text string `url:"text"`
Srv string `url:"srv"`
Id string `url:"sid"`
}
2023-08-15 14:56:57 +05:30
uuidWithHyphen := uuid.New()
uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1)
opt := Options{from + "-" + to, text, "android", uuid + "-0-0"}
v, _ := query.Values(opt)
2023-08-15 14:56:57 +05:30
yandexOut := PostRequest("https://translate.yandex.net/api/v1/tr.json/translate?"+v.Encode(), []byte(""))
gjsonArr := yandexOut.Get("text.0").Array()
var langout LangOut
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "yandex"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
}
2023-08-27 19:54:30 +05:30
func TranslateDeepl(to string, from string, text string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
FromOrig := from
ToOrig := to
2023-07-23 20:00:20 +05:30
var ToValid bool
var FromValid bool
for _, v := range LangListDeepl("sl") {
if v.Id == to {
2023-08-15 14:56:57 +05:30
ToValid = true
2023-07-23 20:00:20 +05:30
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
2023-07-23 20:00:20 +05:30
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
2023-07-23 20:00:20 +05:30
}
answer, err := gdeeplx.Translate(text, from, to, 0)
if err != nil {
return LangOut{}, errors.New("failed")
2023-07-23 20:00:20 +05:30
}
answer1 := answer.(map[string]interface{})
2023-08-15 14:56:57 +05:30
ans := answer1["data"].(string)
var langout LangOut
langout.OutputText = ans
2023-08-16 11:58:03 +05:30
langout.Engine = "deepl"
2023-08-17 19:39:18 +05:30
if from == "auto" {
langout.AutoDetect = strings.ToLower(answer1["detected_lang"].(string))
}
2023-08-16 11:58:03 +05:30
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
return langout, nil
2023-07-23 20:00:20 +05:30
}
2023-08-27 19:54:30 +05:30
2023-08-30 14:41:20 +05:30
func ddgVqdUpdate() {
r, err := http.NewRequest("GET", "https://duckduckgo.com/?q=translate", nil)
if err != nil {
panic(err)
}
UserAgent, ok := os.LookupEnv("MOZHI_USER_AGENT")
if !ok {
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
r.Header.Set("User-Agent", UserAgent)
client := &http.Client{}
res, err := client.Do(r)
if err != nil {
panic(err)
}
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
re := regexp.MustCompile(`vqd="([^"]*)"`)
match := re.FindStringSubmatch(string(body))
ddgVqd = match[1]
}
2023-08-15 22:09:38 +05:30
func TranslateDuckDuckGo(to string, from string, query string) (LangOut, error) {
2023-08-16 11:58:03 +05:30
FromOrig := from
ToOrig := to
2023-08-15 22:09:38 +05:30
var ToValid bool
var FromValid bool
for _, v := range LangListDuckDuckGo("sl") {
if v.Id == to {
ToValid = true
}
if v.Id == from {
FromValid = true
}
if FromValid == true && ToValid == true {
break
}
}
if ToValid != true {
return LangOut{}, errors.New("Target language code invalid")
}
if FromValid != true {
return LangOut{}, errors.New("Source language code invalid")
}
var url string
2023-08-30 14:41:20 +05:30
var langout LangOut
ddgVqdUpdate()
if from == "auto" {
2023-08-30 14:41:20 +05:30
url = "https://duckduckgo.com/translation.js?vqd=" + ddgVqd + "&query=translate&to=" + to
} else {
2023-08-30 14:41:20 +05:30
url = "https://duckduckgo.com/translation.js?vqd=" + ddgVqd + "&query=translate&to=" + to + "&from=" + from
}
duckDuckGoOut := PostRequest(url, []byte(query))
2023-08-15 22:09:38 +05:30
gjsonArr := duckDuckGoOut.Get("translated").Array()
langout.OutputText = gjsonArr[0].String()
2023-08-16 11:58:03 +05:30
langout.Engine = "duckduckgo"
langout.SourceLang = FromOrig
langout.TargetLang = ToOrig
if from == "auto" {
langout.AutoDetect = duckDuckGoOut.Get("detected_language").String()
}
2023-08-15 22:09:38 +05:30
return langout, nil
}
2023-08-27 19:54:30 +05:30
2023-08-17 19:39:18 +05:30
func TranslateAll(to string, from string, query string) []LangOut {
2023-08-28 21:55:00 +05:30
engines := []string{"reverso", "google", "libre", "watson", "mymemory", "yandex", "deepl", "duckduckgo"}
langout := []LangOut{}
for i := 0; i < len(engines); i++ {
data, err := Translate(engines[i], to, from, query)
if err == nil {
langout = append(langout, data)
}
}
2023-08-16 11:58:03 +05:30
return langout
2023-06-25 00:10:57 +05:30
}