diff --git a/go.mod b/go.mod index 174bb78..417d134 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,12 @@ module go.mau.fi/mautrix-twitter go 1.22 require ( + github.com/google/go-querystring v1.1.0 + github.com/google/uuid v1.6.0 + github.com/mattn/go-colorable v0.1.13 + github.com/rs/zerolog v1.33.0 go.mau.fi/util v0.6.0 + golang.org/x/net v0.27.0 maunium.net/go/mautrix v0.19.1-0.20240719130542-cc5f225bc61c ) @@ -11,9 +16,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect - github.com/rs/zerolog v1.33.0 // indirect github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect @@ -22,7 +25,6 @@ require ( go.mau.fi/zeroconfig v0.1.3 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect - golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 09b23da..f2337e0 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,13 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -48,6 +55,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/pkg/twittermeow/README.md b/pkg/twittermeow/README.md new file mode 100644 index 0000000..53f1ebf --- /dev/null +++ b/pkg/twittermeow/README.md @@ -0,0 +1,32 @@ +# x-go +A Go library for interacting with X's API + +### Test steps + +1. Create cookies.txt in this directory, grab your cookie string from x.com and paste it there +2. Run `go test client_test.go -v` + +### Testing functionality + +```go + _, _, err = cli.LoadMessagesPage() + if err != nil { + log.Fatal(err) + } +``` +The `LoadMessagesPage` method makes a request to `https://x.com/messages` then makes 2 calls: +```go + data, err := c.GetAccountSettings(...) + initialInboxState, err := c.GetInitialInboxState(...) +``` +it sets up the current "page" session for the client, fetches the current authenticated user info as well as the initial inbox state (the very starting inbox information you see when u load `/messages`) then returns the parsed data. + +To easily test with the available functions I have made, lets say you wanna test uploading an image and sending it to the top conversation in your inbox you could simply do something like: +```go + initialInboxData, _, err := cli.LoadMessagesPage() + if err != nil { + log.Fatal(err) + } + uploadAndSendImageTest(initialInboxData) +``` +Or feel free to try it out yourself! All the methods are available on the client instance. diff --git a/pkg/twittermeow/account.go b/pkg/twittermeow/account.go new file mode 100644 index 0000000..8fc9108 --- /dev/null +++ b/pkg/twittermeow/account.go @@ -0,0 +1,57 @@ +package twittermeow + +import ( + "encoding/json" + "fmt" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" +) + +func (c *Client) Login() error { + err := c.session.LoadPage(endpoints.BASE_LOGIN_URL) + if err != nil { + return err + } + return nil +} + +func (c *Client) GetAccountSettings(params payload.AccountSettingsQuery) (*response.AccountSettingsResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.ACCOUNT_SETTINGS_URL, string(encodedQuery)) + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.AccountSettingsResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) GetDMPermissions(params payload.GetDMPermissionsQuery) (*response.GetDMPermissionsResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.DM_PERMISSIONS_URL, string(encodedQuery)) + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.GetDMPermissionsResponse{} + return &data, json.Unmarshal(respBody, &data) +} diff --git a/pkg/twittermeow/client.go b/pkg/twittermeow/client.go new file mode 100644 index 0000000..d8618e8 --- /dev/null +++ b/pkg/twittermeow/client.go @@ -0,0 +1,293 @@ +package twittermeow + +import ( + "fmt" + "net" + "net/http" + "net/url" + "strings" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/cookies" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/crypto" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" + + "github.com/rs/zerolog" + "golang.org/x/net/proxy" +) + +type ClientOpts struct { + PollingInterval *time.Duration + Cookies *cookies.Cookies + Session *SessionLoader + EventHandler EventHandler + WithJOTClient bool +} +type EventHandler func(evt interface{}) +type Client struct { + Logger zerolog.Logger + cookies *cookies.Cookies + session *SessionLoader + http *http.Client + httpProxy func(*http.Request) (*url.URL, error) + socksProxy proxy.Dialer + eventHandler EventHandler + + jot *JotClient + polling *PollingClient +} + +func NewClient(opts *ClientOpts, logger zerolog.Logger) *Client { + cli := Client{ + http: &http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 40 * time.Second, + ForceAttemptHTTP2: true, + }, + Timeout: 60 * time.Second, + }, + Logger: logger, + } + + cli.polling = cli.newPollingClient(opts.PollingInterval) + + if opts.WithJOTClient { + cli.jot = cli.newJotClient() + } + + if opts.EventHandler != nil { + cli.SetEventHandler(opts.EventHandler) + } + + if opts.Cookies != nil { + cli.cookies = opts.Cookies + } else { + cli.cookies = cookies.NewCookies() + } + + if opts.Session != nil { + cli.session = opts.Session + } else { + cli.session = cli.newSessionLoader() + } + + return &cli +} + +func (c *Client) Connect() error { + if c.eventHandler == nil { + return ErrConnectPleaseSetEventHandler + } + + if !c.isAuthenticated() { + return ErrNotAuthenticatedYet + } + + return c.polling.startPolling() +} + +func (c *Client) LoadMessagesPage() (*response.XInboxData, *response.AccountSettingsResponse, error) { + err := c.session.LoadPage(endpoints.BASE_MESSAGES_URL) + if err != nil { + return nil, nil, err + } + + data, err := c.GetAccountSettings(payload.AccountSettingsQuery{ + IncludeExtSharingAudiospacesListeningDataWithFollowers: true, + IncludeMentionFilter: true, + IncludeNSFWUserFlag: true, + IncludeNSFWAdminFlag: true, + IncludeRankedTimeline: true, + IncludeAltTextCompose: true, + Ext: "ssoConnections", + IncludeCountryCode: true, + IncludeExtDMNSFWMediaFilter: true, + }) + + if err != nil { + return nil, nil, err + } + + initialInboxState, err := c.GetInitialInboxState((&payload.DmRequestQuery{}).Default()) + if err != nil { + return nil, nil, err + } + + c.session.SetCurrentUser(data) + c.polling.SetCurrentCursor(initialInboxState.InboxInitialState.Cursor) + + c.Logger.Info(). + Str("screen_name", data.ScreenName). + Str("initial_inbox_cursor", initialInboxState.InboxInitialState.Cursor). + Msg("Successfully loaded and authenticated as user") + + return &initialInboxState.InboxInitialState, data, err +} + +func (c *Client) GetCurrentUser() *response.AccountSettingsResponse { + return c.session.GetCurrentUser() +} + +func (c *Client) GetCurrentUserID() string { + twid := c.cookies.Get(cookies.XTwid) + return strings.Replace(twid, "u%3D", "", -1) +} + +func (c *Client) SetProxy(proxyAddr string) error { + proxyParsed, err := url.Parse(proxyAddr) + if err != nil { + return err + } + + if proxyParsed.Scheme == "http" || proxyParsed.Scheme == "https" { + c.httpProxy = http.ProxyURL(proxyParsed) + c.http.Transport.(*http.Transport).Proxy = c.httpProxy + } else if proxyParsed.Scheme == "socks5" { + c.socksProxy, err = proxy.FromURL(proxyParsed, &net.Dialer{Timeout: 20 * time.Second}) + if err != nil { + return err + } + c.http.Transport.(*http.Transport).Dial = c.socksProxy.Dial + contextDialer, ok := c.socksProxy.(proxy.ContextDialer) + if ok { + c.http.Transport.(*http.Transport).DialContext = contextDialer.DialContext + } + } + + c.Logger.Debug(). + Str("scheme", proxyParsed.Scheme). + Str("host", proxyParsed.Host). + Msg("Using proxy") + return nil +} + +func (c *Client) isLoggedIn() bool { + return !c.cookies.IsCookieEmpty(cookies.XAuthToken) +} + +func (c *Client) isAuthenticated() bool { + return c.session.isAuthenticated() +} + +func (c *Client) SetEventHandler(handler EventHandler) { + c.eventHandler = handler +} + +func (c *Client) fetchAndParseMainScript(scriptUrl string) error { + extraHeaders := map[string]string{ + "accept": "*/*", + "sec-fetch-site": "cross-site", + "sec-fetch-mode": "cors", + "sec-fetch-dest": "script", + "origin": endpoints.BASE_URL, + } + _, scriptRespBody, err := c.MakeRequest(scriptUrl, "GET", c.buildHeaders(HeaderOpts{Extra: extraHeaders, Referer: endpoints.BASE_URL + "/"}), nil, types.NONE) + if err != nil { + return err + } + + scriptText := string(scriptRespBody) + + authTokens := methods.ParseBearerTokens(scriptText) + if len(authTokens) < 2 { + return fmt.Errorf("failed to find auth tokens in main script response body") + } + + authenticatedToken, notAuthenticatedToken := authTokens[0], authTokens[1] + c.session.SetAuthTokens(authenticatedToken, notAuthenticatedToken) + + return nil +} + +func (c *Client) parseMainPageHTML(mainPageResp *http.Response, mainPageHTML string) error { + country := methods.ParseCountry(mainPageHTML) + if country == "" { + return fmt.Errorf("failed to find session country by regex in redirected html response body (response_body=%s, status_code=%d)", mainPageHTML, mainPageResp.StatusCode) + } + + verificationToken := methods.ParseVerificationToken(mainPageHTML) + if verificationToken == "" { + return fmt.Errorf("failed to find twitter verification token by regex in redirected html response body (response_body=%s, status_code=%d)", mainPageHTML, mainPageResp.StatusCode) + } + + c.session.SetCountry(country) + c.session.SetVerificationToken(verificationToken) + + guestToken := methods.ParseGuestToken(mainPageHTML) + if guestToken == "" { + if c.cookies.IsCookieEmpty(cookies.XGuestToken) || !c.isLoggedIn() { + // most likely means your cookies are invalid / expired + return fmt.Errorf("failed to find guest token by regex in redirected html response body (response_body=%s, status_code=%d)", mainPageHTML, mainPageResp.StatusCode) + } + } else { + c.cookies.Set(cookies.XGuestToken, guestToken) + } + + mainScriptUrl := methods.ParseMainScriptURL(mainPageHTML) + if mainScriptUrl == "" { + return fmt.Errorf("failed to find main script url by regex in redirected html response body (response_body=%s, status_code=%d)", mainPageHTML, mainPageResp.StatusCode) + } + + err := c.fetchAndParseMainScript(mainScriptUrl) + if err != nil { + return err + } + + return nil +} + +func (c *Client) performJotClientEvent(category payload.JotLoggingCategory, debug bool, body []interface{}) error { + if c.jot == nil { + return nil + } + return c.jot.sendClientLoggingEvent(category, debug, body) +} + +func (c *Client) enableRedirects() { + c.http.CheckRedirect = nil +} + +func (c *Client) disableRedirects() { + c.http.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return ErrRedirectAttempted + } +} + +type apiRequestOpts struct { + Url string + Referer string + Origin string + Method string + Body []byte + ContentType types.ContentType + WithClientUUID bool +} + +func (c *Client) makeAPIRequest(apiRequestOpts apiRequestOpts) (*http.Response, []byte, error) { + clientTransactionId, err := crypto.SignTransaction(c.session.verificationToken, apiRequestOpts.Url, apiRequestOpts.Method) + if err != nil { + return nil, nil, err + } + + headerOpts := HeaderOpts{ + WithAuthBearer: true, + WithCookies: true, + WithXTwitterHeaders: true, + WithXCsrfToken: true, + Referer: apiRequestOpts.Referer, + Origin: apiRequestOpts.Origin, + Extra: map[string]string{ + "x-client-transaction-id": clientTransactionId, + }, + WithXClientUUID: apiRequestOpts.WithClientUUID, + } + headers := c.buildHeaders(headerOpts) + + return c.MakeRequest(apiRequestOpts.Url, apiRequestOpts.Method, headers, apiRequestOpts.Body, apiRequestOpts.ContentType) +} diff --git a/pkg/twittermeow/client_test.go b/pkg/twittermeow/client_test.go new file mode 100644 index 0000000..790740e --- /dev/null +++ b/pkg/twittermeow/client_test.go @@ -0,0 +1,503 @@ +package twittermeow_test + +import ( + "fmt" + "log" + "os" + "testing" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/cookies" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/debug" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/event" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" + + "github.com/google/uuid" +) + +var cli *twittermeow.Client + +func TestXClientLogin(t *testing.T) { + cookieStr, err := os.ReadFile("cookies.txt") + if err != nil { + log.Fatal(err) + } + cookieStruct := cookies.NewCookiesFromString(string(cookieStr)) + + clientOptions := twittermeow.ClientOpts{ + Cookies: cookieStruct, + EventHandler: eventHandler, + WithJOTClient: true, + } + cli = twittermeow.NewClient(&clientOptions, debug.NewLogger()) + cli.SetEventHandler(eventHandler) + + _, _, err = cli.LoadMessagesPage() + if err != nil { + log.Fatal(err) + } + + err = cli.Connect() + if err != nil { + log.Fatal(err) + } + + wait := make(chan struct{}) + <-wait +} + +func deleteConversationTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + payload := payload.DmRequestQuery{}.Default() + err = cli.DeleteConversation(firstConversation.ConversationID, payload) + if err != nil { + log.Fatal(err) + } + + cli.Logger.Info().Str("conversation_id", firstConversation.ConversationID).Msg("Successfully deleted the top conversation in my inbox") +} + +func pinTopConversationTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + pinnedResponse, err := cli.PinConversation(firstConversation.ConversationID) + if err != nil { + log.Fatalf("failed to pin conversation with id: %s (%s)", firstConversation.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("pinnedResponse", pinnedResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Successfully pinned the conversation at the very top of my inbox") +} + +func createConversationAndSendMessageTest() { + searchQuery := payload.SearchQuery{ + IncludeExtIsBlueVerified: "1", + IncludeExtVerifiedType: "1", + IncludeExtProfileImageShape: "1", + Query: "dest", + Src: "compose_message", + ResultType: payload.SEARCH_RESULT_TYPE_USERS, + } + searchResponse, err := cli.Search(searchQuery) + if err != nil { + log.Fatal(err) + } + + var pickedUser *types.User + for _, user := range searchResponse.Users { + if user.IsDmAble { + pickedUser = &user + break + } + } + + if pickedUser == nil { + log.Fatalf("failed to find a user that I can dm while searching for query %s", searchQuery.Query) + } + + dmPermissionsQuery := payload.GetDMPermissionsQuery{ + RecipientIds: pickedUser.IDStr, + DmUsers: true, + } + dmPermissionsResponse, err := cli.GetDMPermissions(dmPermissionsQuery) + if err != nil { + log.Fatalf("failed to fetch dm permissions for recipients with ids %s", dmPermissionsQuery.RecipientIds) + } + + pickedUserDMPermissions := dmPermissionsResponse.Permissions.GetPermissionsForUser(pickedUser.IDStr) + if pickedUserDMPermissions == nil { + log.Fatalf("failed to find permissions for user with id %s", pickedUser.IDStr) + } + + if !pickedUserDMPermissions.CanDm { + log.Fatalf("exiting because I do not have the correct permissions to dm user with id: %s (canDm=%v, errorCode=%d)", pickedUser.IDStr, pickedUserDMPermissions.CanDm, pickedUserDMPermissions.ErrorCode) + } + + myUserID := cli.GetCurrentUserID() + conversationId := fmt.Sprintf("%s-%s", pickedUser.IDStr, myUserID) + + contextQuery := (&payload.DmRequestQuery{}).Default() + contextQuery.IncludeConversationInfo = true + _, err = cli.FetchConversationContext(conversationId, contextQuery, payload.CONTEXT_FETCH_DM_CONVERSATION) + if err != nil { + log.Fatal(err) + } + + sendDirectMessagePayload := &payload.SendDirectMessagePayload{ + ConversationID: conversationId, + CardsPlatform: "Web-12", + IncludeCards: 1, + IncludeQuoteCount: true, + DmUsers: false, + RecipientIds: false, + Text: "testing creating a conversation by sending a message", + } + sentMessageResp, err := cli.SendDirectMessage(sendDirectMessagePayload) + if err != nil { + log.Fatalf("failed to initialize and send message to conversation by id %s", conversationId) + } + + cli.Logger.Info().Any("response", sentMessageResp).Str("conversation_id", conversationId).Str("other_user_id", pickedUser.IDStr).Msg("Successfully initialized new conversation by sending a test message") +} + +func deleteMessageForMeTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + mostRecentMessage := conversations[0].Messages[0] + + payload := &payload.DMMessageDeleteMutationVariables{ + MessageID: mostRecentMessage.MessageData.ID, + } + deleteMessageResp, err := cli.DeleteMessage(payload) + if err != nil { + log.Fatalf("failed to delete message with id %s in conversation with id %s", payload.MessageID, firstConversation.ConversationID) + } + + cli.Logger.Info().Any("deleteMessageResp", deleteMessageResp).Str("conversation_id", firstConversation.ConversationID).Str("message_id", payload.MessageID).Msg("Deleted most recent message in conversation for me") +} + +func uploadAndSendImageTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + // Note: this file doesn't exist + imgBytes, err := os.ReadFile("test_data/testimage1.jpg") + if err != nil { + log.Fatal(err) + } + + uploadQuery := &payload.UploadMediaQuery{ + MediaType: payload.MEDIA_TYPE_IMAGE_JPEG, + MediaCategory: payload.MEDIA_CATEGORY_DM_IMAGE, + } + + mediaResult, err := cli.UploadMedia(uploadQuery, imgBytes) + if err != nil { + log.Fatal(err) + } + + payload := &payload.SendDirectMessagePayload{ + ConversationID: firstConversation.ConversationID, + RequestID: uuid.NewString(), + CardsPlatform: "Web-12", + IncludeCards: 1, + Text: "", + MediaID: mediaResult.MediaIDString, + IncludeQuoteCount: true, + RecipientIds: false, + DmUsers: false, + } + + sentMessageResponse, err := cli.SendDirectMessage(payload) + if err != nil { + log.Fatalf("failed to send image to conversation with id %s (%s)", firstConversation.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("response", sentMessageResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Sent test image to first conversation") +} + +func testReplyToMessage(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + mostRecentMessage := conversations[0].Messages[0] + + payload := &payload.SendDirectMessagePayload{ + ConversationID: firstConversation.ConversationID, + RequestID: uuid.NewString(), + ReplyToDmID: mostRecentMessage.MessageData.ID, + CardsPlatform: "Web-12", + IncludeCards: 1, + Text: "this is a test reply", + IncludeQuoteCount: true, + RecipientIds: false, + DmUsers: false, + } + + sentReplyResponse, err := cli.SendDirectMessage(payload) + if err != nil { + log.Fatalf("failed to reply to message with id %s in conversation with id %s (%s)", mostRecentMessage.MessageData.ID, mostRecentMessage.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("response", sentReplyResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Sent test reply to most recent message in conversation") +} + +func uploadAndSendGifTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + uploadQuery := &payload.UploadMediaQuery{ + SourceURL: "https://media1.giphy.com/media/v1.Y2lkPWU4MjZjOWZjYzdkYTk5YWM3ODE2MjczYTlkYWFiZjY2MDkxYTIyZDJmMjVlMDAwYiZlcD12MV9naWZzX2NhdGVnb3JpZXNfY2F0ZWdvcnlfdGFnJmN0PWc/z3HFoEzXCMykr4L0TB/giphy.gif", + MediaType: payload.MEDIA_TYPE_IMAGE_GIF, + MediaCategory: payload.MEDIA_CATEGORY_DM_GIF, + } + mediaResult, err := cli.UploadMedia(uploadQuery, nil) + if err != nil { + log.Fatal(err) + } + + payload := &payload.SendDirectMessagePayload{ + ConversationID: firstConversation.ConversationID, + RequestID: uuid.NewString(), + CardsPlatform: "Web-12", + IncludeCards: 1, + Text: "", + MediaID: mediaResult.MediaIDString, + IncludeQuoteCount: true, + RecipientIds: false, + DmUsers: false, + } + + sentMessageResponse, err := cli.SendDirectMessage(payload) + if err != nil { + log.Fatalf("failed to send gif to conversation with id %s (%s)", firstConversation.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("response", sentMessageResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Sent test gif to first conversation") +} + +func uploadAndSendVideoTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + // Note: this file doesn't exist + videoBytes, err := os.ReadFile("test_data/testvideo1.mp4") + if err != nil { + log.Fatal(err) + } + + uploadQuery := &payload.UploadMediaQuery{ + MediaType: payload.MEDIA_TYPE_VIDEO_MP4, + MediaCategory: payload.MEDIA_CATEGORY_DM_VIDEO, + } + + mediaResult, err := cli.UploadMedia(uploadQuery, videoBytes) + if err != nil { + log.Fatal(err) + } + + payload := &payload.SendDirectMessagePayload{ + ConversationID: firstConversation.ConversationID, + RequestID: uuid.NewString(), + CardsPlatform: "Web-12", + IncludeCards: 1, + Text: "", + MediaID: mediaResult.MediaIDString, + IncludeQuoteCount: true, + RecipientIds: false, + DmUsers: false, + } + + sentMessageResponse, err := cli.SendDirectMessage(payload) + if err != nil { + log.Fatalf("failed to send video to conversation with id %s (%s)", firstConversation.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("response", sentMessageResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Sent test video to first conversation") +} + +func sendMessageTest(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + firstConversation := conversations[0].Conversation + + payload := &payload.SendDirectMessagePayload{ + ConversationID: firstConversation.ConversationID, + RequestID: uuid.NewString(), + Text: "this is a test message", + CardsPlatform: "Web-12", + IncludeCards: 1, + IncludeQuoteCount: true, + RecipientIds: false, + DmUsers: false, + } + + sentMessageResponse, err := cli.SendDirectMessage(payload) + if err != nil { + log.Fatalf("failed to send msg to conversation with id %s (%s)", firstConversation.ConversationID, err.Error()) + } + + cli.Logger.Info().Any("response", sentMessageResponse).Str("conversation_id", firstConversation.ConversationID).Msg("Sent test message to first conversation") +} + +func logAllTrustedConversations(initialInboxData *response.XInboxData) { + inboxTimelines := initialInboxData.InboxTimelines + trustedInboxTimeline := inboxTimelines.Trusted + + paginationNextEntryID := trustedInboxTimeline.MinEntryID + paginationStatus := trustedInboxTimeline.Status + reqQuery := (&payload.DmRequestQuery{}) + + for paginationStatus == types.HAS_MORE { + reqQuery.MaxID = paginationNextEntryID + nextInboxTimelineResponse, err := cli.FetchTrustedThreads(reqQuery) + if err != nil { + log.Fatal(err) + } + + methods.MergeMaps(initialInboxData.Conversations, nextInboxTimelineResponse.InboxTimeline.Conversations) + methods.MergeMaps(initialInboxData.Users, nextInboxTimelineResponse.InboxTimeline.Users) + initialInboxData.Entries = append(initialInboxData.Entries, nextInboxTimelineResponse.InboxTimeline.Entries...) + + paginationNextEntryID = nextInboxTimelineResponse.InboxTimeline.MinEntryID + paginationStatus = nextInboxTimelineResponse.InboxTimeline.Status + } + + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + + for i, c := range conversations { + conv := c.Conversation + log.Println() + mostRecentMessage := c.Messages[0] + cli.Logger.Info(). + Int("conversation_inbox_position", i). + Str("conversation_id", conv.ConversationID). + Str("type", string(conv.Type)). + Str("createdAt", conv.CreateTime). + Str("createdByUserID", conv.CreatedByUserID). + Any("participants", c.Participants). + Any("most_recent_message", mostRecentMessage.MessageData.Text). + Msg("Inbox Timeline Conversation") + } +} + +func logAllMessagesInConversation(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + + firstConversation := conversations[0].Conversation + + conversationMessageHistoryStatus := firstConversation.Status + if conversationMessageHistoryStatus == types.AT_END { + log.Fatalf("conversation with id %s does not have any more messages to fetch", firstConversation.ConversationID) + } + + totalMessages := len(conversations[0].Messages) + paginationNextEntryID := firstConversation.MinEntryID + reqQuery := (&payload.DmRequestQuery{}).Default() + for conversationMessageHistoryStatus == types.HAS_MORE { + reqQuery.MaxID = paginationNextEntryID + fetchMessagesResponse, err := cli.FetchConversationContext(firstConversation.ConversationID, reqQuery, payload.CONTEXT_FETCH_DM_CONVERSATION_HISTORY) + if err != nil { + log.Fatal(err) + } + + conversationTimeline := fetchMessagesResponse.ConversationTimeline + messageBatch, err := conversationTimeline.PrettifyMessages(firstConversation.ConversationID) + if err != nil { + log.Fatalf("failed to prettify message batch for conversation with id %s (%s)", firstConversation.ConversationID, err.Error()) + } + + for _, msg := range messageBatch { + cli.Logger.Info(). + Str("conversation_id", msg.ConversationID). + Str("sender_name", msg.Sender.Name). + Str("sender_screen_name", msg.Sender.ScreenName). + Str("recipient_name", msg.Recipient.Name). + Str("recipient_screen_name", msg.Recipient.ScreenName). + Str("sent_at", msg.SentAt.String()). + Str("text", msg.Text). + Any("attachment", msg.Attachment). + Any("entities", msg.Entities). + Msg("Message") + } + + totalMessages += len(messageBatch) + conversationMessageHistoryStatus = conversationTimeline.Status + paginationNextEntryID = conversationTimeline.MinEntryID + } + + cli.Logger.Info().Int("total", totalMessages).Str("conversation_id", firstConversation.ConversationID).Msg("Successfully fetched all existing messages in conversation") +} + +func logInitialDisplayFeed(initialInboxData *response.XInboxData) { + conversations, err := initialInboxData.Prettify() + if err != nil { + log.Fatal(err) + } + + for i, c := range conversations { + conv := c.Conversation + mostRecentMessage := c.Messages[0] + cli.Logger.Info(). + Int("conversation_inbox_position", i). + Str("conversation_id", conv.ConversationID). + Str("type", string(conv.Type)). + Str("createdAt", conv.CreateTime). + Str("createdByUserID", conv.CreatedByUserID). + Any("participants", c.Participants). + Any("most_recent_message", mostRecentMessage.MessageData.Text). + Msg("Initial Inbox Display") + } +} + +func eventHandler(evt interface{}) { + switch evtData := evt.(type) { + case event.XEventMessage: + cli.Logger.Info(). + Str("conversation_id", evtData.Conversation.ConversationID). + Str("sender_id", evtData.Sender.IDStr). + Str("recipient_id", evtData.Recipient.IDStr). + Str("message_id", evtData.MessageID). + Str("createdAt", evtData.CreatedAt.String()). + Str("text", evtData.Text). + Any("entities", evtData.Entities). + Any("attachment", evtData.Attachment). + Msg("New message event!") + case event.XEventConversationRead: + cli.Logger.Info(). + Str("conversation_id", evtData.Conversation.ConversationID). + Str("last_read_event_id", evtData.LastReadEventID). + Str("read_at", evtData.ReadAt.String()). + Msg("Conversation was read!") + case event.XEventConversationCreated: + cli.Logger.Info(). + Str("conversation_id", evtData.Conversation.ConversationID). + Any("participants", evtData.Conversation.Participants). + Str("type", string(evtData.Conversation.Type)). + Str("created_at", evtData.CreatedAt.String()). + Msg("New conversation was created!") + case event.XEventMessageDeleted: + cli.Logger.Info(). + Str("conversation_id", evtData.Conversation.ConversationID). + Any("participants", evtData.Conversation.Participants). + Any("messages_deleted", evtData.Messages). + Str("type", string(evtData.Conversation.Type)). + Str("deleted_at", evtData.DeletedAt.String()). + Msg("Messages were deleted!") + default: + log.Println("unknown event:", evt) + } +} diff --git a/pkg/twittermeow/cookies/cookies.go b/pkg/twittermeow/cookies/cookies.go new file mode 100644 index 0000000..c27863a --- /dev/null +++ b/pkg/twittermeow/cookies/cookies.go @@ -0,0 +1,95 @@ +package cookies + +import ( + "fmt" + "net/http" + "strings" + "sync" + "time" +) + +type XCookieName string + +const ( + XAuthToken XCookieName = "auth_token" + XGuestID XCookieName = "guest_id" + XNightMode XCookieName = "night_mode" + XGuestToken XCookieName = "gt" + XCt0 XCookieName = "ct0" + XKdt XCookieName = "kdt" + XTwid XCookieName = "twid" + XLang XCookieName = "lang" + XAtt XCookieName = "att" + XPersonalizationID XCookieName = "personalization_id" + XGuestIDMarketing XCookieName = "guest_id_marketing" +) + +type Cookies struct { + store map[XCookieName]string + lock sync.RWMutex +} + +func NewCookies() *Cookies { + return &Cookies{ + store: make(map[XCookieName]string), + lock: sync.RWMutex{}, + } +} + +func NewCookiesFromString(cookieStr string) *Cookies { + c := NewCookies() + cookieStrings := strings.Split(cookieStr, ";") + fakeHeader := http.Header{} + for _, cookieStr := range cookieStrings { + trimmedCookieStr := strings.TrimSpace(cookieStr) + if trimmedCookieStr != "" { + fakeHeader.Add("Set-Cookie", trimmedCookieStr) + } + } + fakeResponse := &http.Response{Header: fakeHeader} + + for _, cookie := range fakeResponse.Cookies() { + c.store[XCookieName(cookie.Name)] = cookie.Value + } + + return c +} + +func (c *Cookies) String() string { + c.lock.RLock() + defer c.lock.RUnlock() + var out []string + for k, v := range c.store { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(out, "; ") +} + +func (c *Cookies) IsCookieEmpty(key XCookieName) bool { + return c.Get(key) == "" +} + +func (c *Cookies) Get(key XCookieName) string { + c.lock.RLock() + defer c.lock.RUnlock() + return c.store[key] +} + +func (c *Cookies) Set(key XCookieName, value string) { + c.lock.Lock() + defer c.lock.Unlock() + c.store[key] = value +} + +func (c *Cookies) UpdateFromResponse(r *http.Response) { + c.lock.Lock() + defer c.lock.Unlock() + for _, cookie := range r.Cookies() { + if cookie.MaxAge == 0 || cookie.Expires.Before(time.Now()) { + delete(c.store, XCookieName(cookie.Name)) + } else { + //log.Println(fmt.Sprintf("updated cookie %s to value %s", cookie.Name, cookie.Value)) + c.store[XCookieName(cookie.Name)] = cookie.Value + } + } +} diff --git a/pkg/twittermeow/crypto/transaction.go b/pkg/twittermeow/crypto/transaction.go new file mode 100644 index 0000000..41f6b52 --- /dev/null +++ b/pkg/twittermeow/crypto/transaction.go @@ -0,0 +1,155 @@ +package crypto + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "fmt" + "math" + "math/rand" + neturl "net/url" + "strconv" + "strings" + "time" +) + +type TransactionData struct { + VerificationToken string +} + +const ( + timestampConstant = 1682924400000 + // constant color/easing string (i dont think its relevant) + hexStr = "74c5e50fd70a3d70a3d701eb851eb851eb801eb851eb851eb80fd70a3d70a3d700" + twitterStr = "bird" +) + +// it may not be this simple. +// yes this does generate/sign the x-client-transaction header but there might be a lot more to it. +// this doesn't properly grab the svg data embedded in the page, it generates brand new svg data +// there's also some RTC connection involved under the hood which is hard to interpret but I have not investigated it +// the RTC connection being established does send/receive data related to the transaction being signed so it might have something to do with verification + +func SignTransaction(verificationToken, url, method string) (string, error) { + verificationTokenBytes, err := base64.StdEncoding.DecodeString(verificationToken) + if err != nil { + return "", fmt.Errorf("failed to decode verification token while signing client transaction id: %e", err) + } + + parsedUrl, err := neturl.Parse(url) + if err != nil { + return "", fmt.Errorf("failed to parse path for request (url=%s, method=%s): %e", url, method, err) + } + + ts, tsBytes := signTimestamp() + unsignedString := fmt.Sprintf("%s!%s!%s%s%s", method, parsedUrl.Path, ts, twitterStr, hexStr) + + hash := sha256.Sum256([]byte(unsignedString)) + hashSlice := hash[:16] + + resultBytes := bytes.NewBuffer([]byte{}) + resultBytes.WriteByte(byte(randNum())) + resultBytes.Write(verificationTokenBytes) + resultBytes.Write(tsBytes) + resultBytes.Write(hashSlice) + resultBytes.WriteByte(1) + + signedBytes := encodeXOR(resultBytes.Bytes()) + return base64.RawStdEncoding.EncodeToString(signedBytes), nil +} + +func randNum() int { + return rand.Intn(256) +} + +func signTimestamp() (string, []byte) { + elapsed := time.Now().UnixNano()/int64(time.Millisecond) - timestampConstant + result := int(math.Floor(float64(elapsed) / 1000.0)) + resultInt32 := make([]byte, 4) + + binary.LittleEndian.PutUint32(resultInt32, uint32(result)) + + return strconv.Itoa(result), resultInt32 +} + +func encodeXOR(plainArr []byte) []byte { + encodedArr := make([]byte, len(plainArr)) + for i := 0; i < len(plainArr); i++ { + if i == 0 { + encodedArr[i] = plainArr[i] + } else { + encodedArr[i] = plainArr[i] ^ plainArr[0] + } + } + return encodedArr +} + +func decodeSVGStr(svgStr string) [][]int { + segmentStrings := strings.Split(strings.TrimSpace(svgStr), "C")[1:] + byteArrays := [][]int{} + + for _, segment := range segmentStrings { + segment = strings.ReplaceAll(strings.ReplaceAll(segment, "h", ""), "s", "") + coords := strings.Fields(segment) + bytesList := []int{} + + for _, coord := range coords { + if strings.Contains(coord, ",") { + for _, s := range strings.Split(coord, ",") { + num, err := strconv.Atoi(s) + if err == nil { + bytesList = append(bytesList, num) + } + } + } else { + num, err := strconv.Atoi(coord) + if err == nil { + bytesList = append(bytesList, num) + } + } + } + byteArrays = append(byteArrays, bytesList) + } + + return byteArrays +} + +func buildColorStr(arr []int) []string { + colors := []string{} + s := "" + for i := 0; i < 3; i++ { + if arr[i] < 16 { + s += "0" + } + s += strconv.FormatInt(int64(arr[i]), 16) + } + colors = append(colors, s) + s = "" + for i := 3; i < 6; i++ { + if arr[i] < 16 { + s += "0" + } + s += strconv.FormatInt(int64(arr[i]), 16) + } + colors = append(colors, s) + return colors +} + +func buildEasingStr(arr []int) []float64 { + nums := []float64{} + t := 1.0 + + for i := 0; i < len(arr); i++ { + b := float64(arr[i]) + o := 0.0 + if i%2 != 0 { + o = -1.0 + } + val := (((t - o) * b) / 255.0) + o + roundedVal, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", val), 64) + nums = append(nums, roundedVal) + } + + return nums +} diff --git a/pkg/twittermeow/crypto/transaction_test.go b/pkg/twittermeow/crypto/transaction_test.go new file mode 100644 index 0000000..da5be9a --- /dev/null +++ b/pkg/twittermeow/crypto/transaction_test.go @@ -0,0 +1,17 @@ +package crypto_test + +import ( + "log" + "testing" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/crypto" +) + +func TestXClientTransactionId(t *testing.T) { + verificationToken := "" + v, err := crypto.SignTransaction(verificationToken, "/1.1/jot/client_event.json", "POST") + if err != nil { + log.Fatalf("failed to sign client transaction id: %s", err.Error()) + } + log.Println(v) +} diff --git a/pkg/twittermeow/data/endpoints/endpoints.go b/pkg/twittermeow/data/endpoints/endpoints.go new file mode 100644 index 0000000..5f757ee --- /dev/null +++ b/pkg/twittermeow/data/endpoints/endpoints.go @@ -0,0 +1,37 @@ +package endpoints + +const ( + TWITTER_BASE_HOST = "twitter.com" + TWITTER_BASE_URL = "https://" + TWITTER_BASE_HOST + + BASE_HOST = "x.com" + BASE_URL = "https://" + BASE_HOST + BASE_LOGIN_URL = BASE_URL + "/login" + BASE_MESSAGES_URL = BASE_URL + "/messages" + + API_BASE_HOST = "api.x.com" + API_BASE_URL = "https://" + API_BASE_HOST + + ACCOUNT_SETTINGS_URL = API_BASE_URL + "/1.1/account/settings.json" + INBOX_INITIAL_STATE_URL = BASE_URL + "/i/api/1.1/dm/inbox_initial_state.json" + DM_USER_UPDATES_URL = BASE_URL + "/i/api/1.1/dm/user_updates.json" + CONVERSATION_MARK_READ_URL = BASE_URL + "/i/api/1.1/dm/conversation/%s/mark_read.json" + CONVERSATION_FETCH_MESSAGES = BASE_URL + "/i/api/1.1/dm/conversation/%s.json" + UPDATE_LAST_SEEN_EVENT_ID_URL = BASE_URL + "/i/api/1.1/dm/update_last_seen_event_id.json" + TRUSTED_INBOX_TIMELINE_URL = BASE_URL + "/i/api/1.1/dm/inbox_timeline/trusted.json" + SEND_DM_URL = BASE_URL + "/i/api/1.1/dm/new2.json" + GRAPHQL_MESSAGE_DELETION_MUTATION = BASE_URL + "/i/api/graphql/BJ6DtxA2llfjnRoRjaiIiw/DMMessageDeleteMutation" + SEARCH_TYPEAHEAD_URL = BASE_URL + "/i/api/1.1/search/typeahead.json" + DM_PERMISSIONS_URL = BASE_URL + "/i/api/1.1/dm/permissions.json" + DELETE_CONVERSATION_URL = BASE_URL + "/i/api/1.1/dm/conversation/%s/delete.json" + PIN_CONVERSATION_URL = BASE_URL + "/i/api/graphql/o0aymgGiJY-53Y52YSUGVA/DMPinnedInboxAppend_Mutation" + UNPIN_CONVERSATION_URL = BASE_URL + "/i/api/graphql/_TQxP2Rb0expwVP9ktGrTQ/DMPinnedInboxDelete_Mutation" + GET_PINNED_CONVERSATIONS_URL = BASE_URL + "/i/api/graphql/_gBQBgClVuMQb8efxWkbbQ/DMPinnedInboxQuery" + + JOT_CLIENT_EVENT_URL = API_BASE_URL + "/1.1/jot/client_event.json" + JOT_CES_P2_URL = API_BASE_URL + "/1.1/jot/ces/p2" + + UPLOAD_BASE_HOST = "upload.x.com" + UPLOAD_BASE_URL = "https://" + UPLOAD_BASE_HOST + UPLOAD_MEDIA_URL = UPLOAD_BASE_URL + "/i/media/upload.json" +) diff --git a/pkg/twittermeow/data/payload/form.go b/pkg/twittermeow/data/payload/form.go new file mode 100644 index 0000000..b9b335c --- /dev/null +++ b/pkg/twittermeow/data/payload/form.go @@ -0,0 +1,218 @@ +package payload + +import "github.com/google/go-querystring/query" + +type MigrationRequestPayload struct { + Tok string `url:"tok"` + Data string `url:"data"` +} + +type JotClientEventPayload struct { + Category JotLoggingCategory `url:"category,omitempty"` + Debug bool `url:"debug,omitempty"` + Log string `url:"log"` +} + +func (p *JotClientEventPayload) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +type AccountSettingsQuery struct { + IncludeExtSharingAudiospacesListeningDataWithFollowers bool `url:"include_ext_sharing_audiospaces_listening_data_with_followers"` + IncludeMentionFilter bool `url:"include_mention_filter"` + IncludeNSFWUserFlag bool `url:"include_nsfw_user_flag"` + IncludeNSFWAdminFlag bool `url:"include_nsfw_admin_flag"` + IncludeRankedTimeline bool `url:"include_ranked_timeline"` + IncludeAltTextCompose bool `url:"include_alt_text_compose"` + Ext string `url:"ext"` + IncludeCountryCode bool `url:"include_country_code"` + IncludeExtDMNSFWMediaFilter bool `url:"include_ext_dm_nsfw_media_filter"` +} + +func (p *AccountSettingsQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +type ContextInfo string + +const ( + CONTEXT_FETCH_DM_CONVERSATION ContextInfo = "FETCH_DM_CONVERSATION" + CONTEXT_FETCH_DM_CONVERSATION_HISTORY ContextInfo = "FETCH_DM_CONVERSATION_HISTORY" +) + +type DmRequestQuery struct { + ActiveConversationId string `url:"active_conversation_id,omitempty"` + Cursor string `url:"cursor,omitempty"` + Context ContextInfo `url:"context,omitempty"` + MaxID string `url:"max_id,omitempty"` // when fetching messages, this is the message id + NSFWFilteringEnabled bool `url:"nsfw_filtering_enabled"` + IncludeProfileInterstitialType int `url:"include_profile_interstitial_type"` + IncludeBlocking int `url:"include_blocking"` + IncludeConversationInfo bool `url:"include_conversation_info"` + IncludeBlockedBy int `url:"include_blocked_by"` + IncludeFollowedBy int `url:"include_followed_by"` + IncludeWantRetweets int `url:"include_want_retweets"` + IncludeMuteEdge int `url:"include_mute_edge"` + IncludeCanDM int `url:"include_can_dm"` + IncludeCanMediaTag int `url:"include_can_media_tag"` + IncludeExtIsBlueVerified int `url:"include_ext_is_blue_verified"` + IncludeExtVerifiedType int `url:"include_ext_verified_type"` + IncludeExtProfileImageShape int `url:"include_ext_profile_image_shape"` + SkipStatus int `url:"skip_status"` + DMSecretConversationsEnabled bool `url:"dm_secret_conversations_enabled"` + KRSRegistrationEnabled bool `url:"krs_registration_enabled"` + CardsPlatform string `url:"cards_platform"` + IncludeCards int `url:"include_cards"` + IncludeExtAltText bool `url:"include_ext_alt_text"` + IncludeExtLimitedActionResults bool `url:"include_ext_limited_action_results"` + IncludeQuoteCount bool `url:"include_quote_count"` + IncludeReplyCount int `url:"include_reply_count"` + TweetMode string `url:"tweet_mode"` + IncludeExtViews bool `url:"include_ext_views"` + DMUsers bool `url:"dm_users"` + IncludeGroups bool `url:"include_groups"` + IncludeInboxTimelines bool `url:"include_inbox_timelines"` + IncludeExtMediaColor bool `url:"include_ext_media_color"` + SupportsReactions bool `url:"supports_reactions"` + IncludeExtEditControl bool `url:"include_ext_edit_control"` + IncludeExtBusinessAffiliationsLabel bool `url:"include_ext_business_affiliations_label"` + Ext string `url:"ext"` +} + +func (p *DmRequestQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +func (p DmRequestQuery) Default() *DmRequestQuery { + return &DmRequestQuery{ + NSFWFilteringEnabled: false, + IncludeProfileInterstitialType: 1, + IncludeBlocking: 1, + IncludeBlockedBy: 1, + IncludeFollowedBy: 1, + IncludeWantRetweets: 1, + IncludeMuteEdge: 1, + IncludeCanDM: 1, + IncludeCanMediaTag: 1, + IncludeExtIsBlueVerified: 1, + IncludeExtVerifiedType: 1, + IncludeExtProfileImageShape: 1, + SkipStatus: 1, + DMSecretConversationsEnabled: false, + KRSRegistrationEnabled: true, + CardsPlatform: "Web-12", + IncludeCards: 1, + IncludeExtAltText: true, + IncludeExtLimitedActionResults: true, + IncludeQuoteCount: true, + IncludeReplyCount: 1, + TweetMode: "extended", + IncludeExtViews: true, + DMUsers: true, + IncludeGroups: true, + IncludeInboxTimelines: true, + IncludeExtMediaColor: true, + SupportsReactions: true, + IncludeExtEditControl: true, + IncludeExtBusinessAffiliationsLabel: true, + Ext: "mediaColor,altText,mediaStats,highlightedLabel,voiceInfo,birdwatchPivot,superFollowMetadata,unmentionInfo,editControl,article", + } +} + +type MarkConversationReadQuery struct { + ConversationID string `url:"conversationId"` + LastReadEventID string `url:"last_read_event_id"` +} + +func (p *MarkConversationReadQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +type MediaCategory string + +const ( + MEDIA_CATEGORY_DM_IMAGE MediaCategory = "dm_image" + MEDIA_CATEGORY_DM_VIDEO MediaCategory = "dm_video" + MEDIA_CATEGORY_DM_GIF MediaCategory = "dm_gif" +) + +type MediaType string + +const ( + MEDIA_TYPE_IMAGE_JPEG MediaType = "image/jpeg" + MEDIA_TYPE_IMAGE_GIF MediaType = "image/gif" + MEDIA_TYPE_VIDEO_MP4 MediaType = "video/mp4" +) + +type UploadMediaQuery struct { + Command string `url:"command,omitempty"` + TotalBytes int `url:"total_bytes,omitempty"` + SourceURL string `url:"source_url,omitempty"` + MediaID string `url:"media_id,omitempty"` + VideoDurationMS float32 `url:"video_duration_ms,omitempty"` + OriginalMd5 string `url:"original_md5,omitempty"` + SegmentIndex int `url:"segment_index,omitempty"` + MediaType MediaType `url:"media_type,omitempty"` + MediaCategory MediaCategory `url:"media_category,omitempty"` +} + +func (p *UploadMediaQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +type SearchResultType string + +const ( + SEARCH_RESULT_TYPE_USERS SearchResultType = "users" +) + +type SearchQuery struct { + IncludeExtIsBlueVerified string `url:"include_ext_is_blue_verified"` + IncludeExtVerifiedType string `url:"include_ext_verified_type"` + IncludeExtProfileImageShape string `url:"include_ext_profile_image_shape"` + Query string `url:"q"` + Src string `url:"src"` + ResultType SearchResultType `url:"result_type"` +} + +func (p *SearchQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} + +type GetDMPermissionsQuery struct { + // seperated by commas: userid1,userid2,userid3 + RecipientIds string `url:"recipient_ids"` + DmUsers bool `url:"dm_users"` +} + +func (p *GetDMPermissionsQuery) Encode() ([]byte, error) { + values, err := query.Values(p) + if err != nil { + return nil, err + } + return []byte(values.Encode()), nil +} diff --git a/pkg/twittermeow/data/payload/jot.go b/pkg/twittermeow/data/payload/jot.go new file mode 100644 index 0000000..7f96c97 --- /dev/null +++ b/pkg/twittermeow/data/payload/jot.go @@ -0,0 +1,44 @@ +package payload + +import "encoding/json" + +type JotLoggingCategory string + +const ( + JotLoggingCategoryPerftown JotLoggingCategory = "perftown" +) + +type JotLogPayload struct { + Description string `json:"description,omitempty"` + Product string `json:"product,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + EventValue int64 `json:"event_value,omitempty"` +} + +func (p *JotLogPayload) ToJSON() ([]byte, error) { + val := []interface{}{p} + return json.Marshal(&val) +} + +type JotDebugLoggingCategory string + +const ( + JotDebugLoggingCategoryClientEvent JotDebugLoggingCategory = "client_event" +) + +type JotDebugLogPayload struct { + Category JotDebugLoggingCategory `json:"_category_,omitempty"` + FormatVersion int `json:"format_version,omitempty"` + TriggeredOn int64 `json:"triggered_on,omitempty"` + Items []any `json:"items,omitempty"` + EventNamespace EventNamespace `json:"event_namespace,omitempty"` + ClientEventSequenceStartTimestamp int64 `json:"client_event_sequence_start_timestamp,omitempty"` + ClientEventSequenceNumber int `json:"client_event_sequence_number,omitempty"` + ClientAppID string `json:"client_app_id,omitempty"` +} + +type EventNamespace struct { + Page string `json:"page,omitempty"` + Action string `json:"action,omitempty"` + Client string `json:"client,omitempty"` +} diff --git a/pkg/twittermeow/data/payload/json.go b/pkg/twittermeow/data/payload/json.go new file mode 100644 index 0000000..daad40e --- /dev/null +++ b/pkg/twittermeow/data/payload/json.go @@ -0,0 +1,46 @@ +package payload + +import "encoding/json" + +type SendDirectMessagePayload struct { + ConversationID string `json:"conversation_id,omitempty"` + MediaID string `json:"media_id,omitempty"` + ReplyToDmID string `json:"reply_to_dm_id,omitempty"` + RecipientIds bool `json:"recipient_ids"` + RequestID string `json:"request_id,omitempty"` + Text string `json:"text"` + CardsPlatform string `json:"cards_platform,omitempty"` + IncludeCards int `json:"include_cards,omitempty"` + IncludeQuoteCount bool `json:"include_quote_count"` + DmUsers bool `json:"dm_users"` +} + +func (p *SendDirectMessagePayload) Encode() ([]byte, error) { + return json.Marshal(p) +} + +type GraphQLPayload struct { + Variables interface{} `json:"variables,omitempty"` + QueryID string `json:"queryId,omitempty"` +} + +func (p *GraphQLPayload) Encode() ([]byte, error) { + return json.Marshal(p) +} + +type DMMessageDeleteMutationVariables struct { + MessageID string `json:"messageId,omitempty"` + RequestID string `json:"requestId,omitempty"` +} + +type LabelType string + +const ( + LABEL_TYPE_PINNED LabelType = "Pinned" +) + +type PinAndUnpinConversationVariables struct { + ConversationID string `json:"conversation_id,omitempty"` + LabelType LabelType `json:"label_type,omitempty"` + Label LabelType `json:"label,omitempty"` +} diff --git a/pkg/twittermeow/data/response/account.go b/pkg/twittermeow/data/response/account.go new file mode 100644 index 0000000..9bacf27 --- /dev/null +++ b/pkg/twittermeow/data/response/account.go @@ -0,0 +1,90 @@ +package response + +import "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + +type AccountSettingsResponse struct { + Protected bool `json:"protected,omitempty"` + ScreenName string `json:"screen_name,omitempty"` + AlwaysUseHTTPS bool `json:"always_use_https,omitempty"` + UseCookiePersonalization bool `json:"use_cookie_personalization,omitempty"` + SleepTime SleepTime `json:"sleep_time,omitempty"` + GeoEnabled bool `json:"geo_enabled,omitempty"` + Language string `json:"language,omitempty"` + DiscoverableByEmail bool `json:"discoverable_by_email,omitempty"` + DiscoverableByMobilePhone bool `json:"discoverable_by_mobile_phone,omitempty"` + DisplaySensitiveMedia bool `json:"display_sensitive_media,omitempty"` + PersonalizedTrends bool `json:"personalized_trends,omitempty"` + AllowMediaTagging string `json:"allow_media_tagging,omitempty"` + AllowContributorRequest string `json:"allow_contributor_request,omitempty"` + AllowAdsPersonalization bool `json:"allow_ads_personalization,omitempty"` + AllowLoggedOutDevicePersonalization bool `json:"allow_logged_out_device_personalization,omitempty"` + AllowLocationHistoryPersonalization bool `json:"allow_location_history_personalization,omitempty"` + AllowSharingDataForThirdPartyPersonalization bool `json:"allow_sharing_data_for_third_party_personalization,omitempty"` + AllowDmsFrom string `json:"allow_dms_from,omitempty"` + AlwaysAllowDmsFromSubscribers any `json:"always_allow_dms_from_subscribers,omitempty"` + AllowDmGroupsFrom string `json:"allow_dm_groups_from,omitempty"` + TranslatorType string `json:"translator_type,omitempty"` + CountryCode string `json:"country_code,omitempty"` + NsfwUser bool `json:"nsfw_user,omitempty"` + NsfwAdmin bool `json:"nsfw_admin,omitempty"` + RankedTimelineSetting any `json:"ranked_timeline_setting,omitempty"` + RankedTimelineEligible any `json:"ranked_timeline_eligible,omitempty"` + AddressBookLiveSyncEnabled bool `json:"address_book_live_sync_enabled,omitempty"` + UniversalQualityFilteringEnabled string `json:"universal_quality_filtering_enabled,omitempty"` + DmReceiptSetting string `json:"dm_receipt_setting,omitempty"` + AltTextComposeEnabled any `json:"alt_text_compose_enabled,omitempty"` + MentionFilter string `json:"mention_filter,omitempty"` + AllowAuthenticatedPeriscopeRequests bool `json:"allow_authenticated_periscope_requests,omitempty"` + ProtectPasswordReset bool `json:"protect_password_reset,omitempty"` + RequirePasswordLogin bool `json:"require_password_login,omitempty"` + RequiresLoginVerification bool `json:"requires_login_verification,omitempty"` + ExtSharingAudiospacesListeningDataWithFollowers bool `json:"ext_sharing_audiospaces_listening_data_with_followers,omitempty"` + Ext Ext `json:"ext,omitempty"` + DmQualityFilter string `json:"dm_quality_filter,omitempty"` + AutoplayDisabled bool `json:"autoplay_disabled,omitempty"` + SettingsMetadata SettingsMetadata `json:"settings_metadata,omitempty"` +} +type SleepTime struct { + Enabled bool `json:"enabled,omitempty"` + EndTime any `json:"end_time,omitempty"` + StartTime any `json:"start_time,omitempty"` +} +type Ok struct { + SsoIDHash string `json:"ssoIdHash,omitempty"` + SsoProvider string `json:"ssoProvider,omitempty"` +} +type R struct { + Ok []Ok `json:"ok,omitempty"` +} +type SsoConnections struct { + R R `json:"r,omitempty"` + TTL int `json:"ttl,omitempty"` +} +type Ext struct { + SsoConnections SsoConnections `json:"ssoConnections,omitempty"` +} +type SettingsMetadata struct { + IsEu string `json:"is_eu,omitempty"` +} + +type GetDMPermissionsResponse struct { + Permissions Permissions `json:"permissions,omitempty"` + Users map[string]types.User `json:"users,omitempty"` +} + +type PermissionDetails struct { + CanDm bool `json:"can_dm,omitempty"` + ErrorCode int `json:"error_code,omitempty"` +} + +type Permissions struct { + IDKeys map[string]PermissionDetails `json:"id_keys,omitempty"` +} + +func (perms Permissions) GetPermissionsForUser(userId string) *PermissionDetails { + if user, ok := perms.IDKeys[userId]; ok { + return &user + } + + return nil +} diff --git a/pkg/twittermeow/data/response/events.go b/pkg/twittermeow/data/response/events.go new file mode 100644 index 0000000..fbbc351 --- /dev/null +++ b/pkg/twittermeow/data/response/events.go @@ -0,0 +1,162 @@ +package response + +import ( + "encoding/json" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/event" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" +) + +type GetDMUserUpdatesResponse struct { + InboxInitialState XInboxData `json:"inbox_initial_state,omitempty"` + UserEvents XInboxData `json:"user_events,omitempty"` +} + +type UserEvents struct { + MinEntryID string `json:"min_entry_id,omitempty"` + MaxEntryID string `json:"max_entry_id,omitempty"` + Cursor string `json:"cursor,omitempty"` + LastSeenEventID string `json:"last_seen_event_id,omitempty"` + TrustedLastSeenEventID string `json:"trusted_last_seen_event_id,omitempty"` + UntrustedLastSeenEventID string `json:"untrusted_last_seen_event_id,omitempty"` + Entries []map[event.XEventType]interface{} `json:"entries,omitempty"` + Users map[string]types.User `json:"users,omitempty"` + Conversations map[string]types.Conversation `json:"conversations,omitempty"` +} + +func (data *XInboxData) GetUserByID(userId string) types.User { + if user, ok := data.Users[userId]; ok { + return user + } + return types.User{} +} + +func (data *XInboxData) GetConversationByID(conversationId string) types.Conversation { + if conv, ok := data.Conversations[conversationId]; ok { + return conv + } + return types.Conversation{} +} + +func (data *XInboxData) ToEventEntries() ([]interface{}, error) { + // entries := make([]map[event.XEventType]interface{}, 0) + entries := make([]interface{}, 0) + if len(data.Entries) <= 0 { + return entries, nil + } + + for _, entry := range data.Entries { + for entryType, entryData := range entry { + var updatedEntry interface{} + jsonEvData, err := json.Marshal(entryData) + if err != nil { + return nil, err + } + switch entryType { + case event.XMessageEvent: + var messageEventData types.Message + err = json.Unmarshal(jsonEvData, &messageEventData) + if err != nil { + return nil, err + } + + createdAt, err := methods.UnixStringMilliToTime(messageEventData.MessageData.Time) + if err != nil { + return nil, err + } + + updatedEntry = event.XEventMessage{ + Conversation: data.GetConversationByID(messageEventData.ConversationID), + Sender: data.GetUserByID(messageEventData.MessageData.SenderID), + Recipient: data.GetUserByID(messageEventData.MessageData.RecipientID), + MessageID: messageEventData.MessageData.ID, + CreatedAt: createdAt, + Text: messageEventData.MessageData.Text, + Entities: messageEventData.MessageData.Entities, + Attachment: messageEventData.MessageData.Attachment, + AffectsSort: messageEventData.AffectsSort, + } + case event.XMessageDeleteEvent: + var messageDeletedEventData types.MessageDeleted + err = json.Unmarshal(jsonEvData, &messageDeletedEventData) + if err != nil { + return nil, err + } + + deletedAt, err := methods.UnixStringMilliToTime(messageDeletedEventData.Time) + if err != nil { + return nil, err + } + + updatedEntry = event.XEventMessageDeleted{ + Conversation: data.GetConversationByID(messageDeletedEventData.ConversationID), + DeletedAt: deletedAt, + EventID: messageDeletedEventData.ID, + RequestID: messageDeletedEventData.RequestID, + AffectsSort: messageDeletedEventData.AffectsSort, + Messages: messageDeletedEventData.Messages, + } + case event.XConversationReadEvent: + var convReadEventData types.ConversationRead + err = json.Unmarshal(jsonEvData, &convReadEventData) + if err != nil { + return nil, err + } + + readAt, err := methods.UnixStringMilliToTime(convReadEventData.Time) + if err != nil { + return nil, err + } + + updatedEntry = event.XEventConversationRead{ + EventID: convReadEventData.ID, + Conversation: data.GetConversationByID(convReadEventData.ConversationID), + ReadAt: readAt, + AffectsSort: convReadEventData.AffectsSort, + LastReadEventID: convReadEventData.LastReadEventID, + } + case event.XConversationCreateEvent: + var convCreatedEventData types.ConversationCreatedData + err = json.Unmarshal(jsonEvData, &convCreatedEventData) + if err != nil { + return nil, err + } + + createdAt, err := methods.UnixStringMilliToTime(convCreatedEventData.Time) + if err != nil { + return nil, err + } + + updatedEntry = event.XEventConversationCreated{ + EventID: convCreatedEventData.ID, + Conversation: data.GetConversationByID(convCreatedEventData.ConversationID), + CreatedAt: createdAt, + AffectsSort: convCreatedEventData.AffectsSort, + RequestID: convCreatedEventData.RequestID, + } + case event.XConversationMetadataUpdateEvent: + var convMetadataUpdateEventData types.ConversationMetadataUpdate + err = json.Unmarshal(jsonEvData, &convMetadataUpdateEventData) + if err != nil { + return nil, err + } + + updatedAt, err := methods.UnixStringMilliToTime(convMetadataUpdateEventData.Time) + if err != nil { + return nil, err + } + + updatedEntry = event.XEventConversationMetadataUpdate{ + EventID: convMetadataUpdateEventData.ID, + Conversation: data.GetConversationByID(convMetadataUpdateEventData.ConversationID), + UpdatedAt: updatedAt, + AffectsSort: convMetadataUpdateEventData.AffectsSort, + } + } + entries = append(entries, updatedEntry) + } + } + + return entries, nil +} diff --git a/pkg/twittermeow/data/response/inbox.go b/pkg/twittermeow/data/response/inbox.go new file mode 100644 index 0000000..0ab91ca --- /dev/null +++ b/pkg/twittermeow/data/response/inbox.go @@ -0,0 +1,133 @@ +package response + +import ( + "encoding/json" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/event" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" +) + +type XInboxData struct { + Status types.PaginationStatus `json:"status,omitempty"` + MinEntryID string `json:"min_entry_id,omitempty"` + MaxEntryID string `json:"max_entry_id,omitempty"` + LastSeenEventID string `json:"last_seen_event_id,omitempty"` + TrustedLastSeenEventID string `json:"trusted_last_seen_event_id,omitempty"` + UntrustedLastSeenEventID string `json:"untrusted_last_seen_event_id,omitempty"` + Cursor string `json:"cursor,omitempty"` + InboxTimelines InboxTimelines `json:"inbox_timelines,omitempty"` + Entries []map[event.XEventType]interface{} `json:"entries,omitempty"` + Users map[string]types.User `json:"users,omitempty"` + Conversations map[string]types.Conversation `json:"conversations,omitempty"` + KeyRegistryState KeyRegistryState `json:"key_registry_state,omitempty"` +} + +type XInboxConversationFeedData struct { + Conversation types.Conversation + Participants []types.User + // sorted by timestamp, first index is the most recent message + Messages []types.Message +} + +func (data *XInboxData) Prettify() ([]XInboxConversationFeedData, error) { + conversationFeeds := make([]XInboxConversationFeedData, 0) + sortedConversations := methods.SortConversationsByTimestamp(data.Conversations) + + for _, conv := range sortedConversations { + messages, err := data.GetMessageEntriesByConversationID(conv.ConversationID, true) + if err != nil { + return nil, err + } + + feedData := XInboxConversationFeedData{ + Conversation: conv, + Participants: data.GetParticipantUsers(conv.Participants), + Messages: messages, + } + conversationFeeds = append(conversationFeeds, feedData) + } + + return conversationFeeds, nil +} + +type PrettifiedMessage struct { + EventID string + ConversationID string + MessageID string + Recipient types.User + Sender types.User + SentAt time.Time + AffectsSort bool + Text string + Attachment types.Attachment + Entities types.Entities +} + +func (data *XInboxData) PrettifyMessages(conversationId string) ([]PrettifiedMessage, error) { + messages, err := data.GetMessageEntriesByConversationID(conversationId, true) + if err != nil { + return nil, err + } + + prettifiedMessages := make([]PrettifiedMessage, 0) + for _, msg := range messages { + sentAt, err := methods.UnixStringMilliToTime(msg.Time) + if err != nil { + return nil, err + } + + prettifiedMessage := PrettifiedMessage{ + EventID: msg.ID, + ConversationID: msg.ConversationID, + MessageID: msg.MessageData.ID, + Sender: data.GetUserByID(msg.MessageData.SenderID), + Recipient: data.GetUserByID(msg.MessageData.RecipientID), + SentAt: sentAt, + Text: msg.MessageData.Text, + Attachment: msg.MessageData.Attachment, + Entities: msg.MessageData.Entities, + AffectsSort: msg.AffectsSort, + } + prettifiedMessages = append(prettifiedMessages, prettifiedMessage) + } + + return prettifiedMessages, nil +} + +func (data *XInboxData) GetParticipantUsers(participants []types.Participants) []types.User { + result := make([]types.User, 0) + for _, participant := range participants { + result = append(result, data.GetUserByID(participant.UserID)) + } + return result +} + +func (data *XInboxData) GetMessageEntriesByConversationID(conversationId string, sortByTimestamp bool) ([]types.Message, error) { + messages := make([]types.Message, 0) + for _, entry := range data.Entries { + for entryType, entryData := range entry { + if entryType == event.XMessageEvent { + jsonEvData, err := json.Marshal(entryData) + if err != nil { + return nil, err + } + var message types.Message + err = json.Unmarshal(jsonEvData, &message) + if err != nil { + return nil, err + } + + if message.ConversationID == conversationId { + messages = append(messages, message) + } + } + } + } + + if sortByTimestamp { + methods.SortMessagesByTime(messages) + } + return messages, nil +} diff --git a/pkg/twittermeow/data/response/media.go b/pkg/twittermeow/data/response/media.go new file mode 100644 index 0000000..85e9540 --- /dev/null +++ b/pkg/twittermeow/data/response/media.go @@ -0,0 +1,43 @@ +package response + +type InitUploadMediaResponse struct { + MediaID int64 `json:"media_id,omitempty"` + MediaIDString string `json:"media_id_string,omitempty"` + ExpiresAfterSecs int `json:"expires_after_secs,omitempty"` + MediaKey string `json:"media_key,omitempty"` +} + +type FinalizedUploadMediaResponse struct { + MediaID int64 `json:"media_id,omitempty"` + MediaIDString string `json:"media_id_string,omitempty"` + MediaKey string `json:"media_key,omitempty"` + Size int `json:"size,omitempty"` + ExpiresAfterSecs int `json:"expires_after_secs,omitempty"` + Image Image `json:"image,omitempty"` + Video Video `json:"video,omitempty"` + ProcessingInfo ProcessingInfo `json:"processing_info,omitempty"` +} + +type Image struct { + ImageType string `json:"image_type,omitempty"` + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` +} + +type Video struct { + VideoType string `json:"video_type,omitempty"` +} + +type ProcessingState string + +const ( + PROCESSING_STATE_PENDING ProcessingState = "pending" + PROCESSING_STATE_IN_PROGRESS ProcessingState = "in_progress" + PROCESSING_STATE_SUCCEEDED ProcessingState = "succeeded" +) + +type ProcessingInfo struct { + State ProcessingState `json:"state,omitempty"` + CheckAfterSecs int `json:"check_after_secs,omitempty"` + ProgressPercent int `json:"progress_percent,omitempty"` +} diff --git a/pkg/twittermeow/data/response/messaging.go b/pkg/twittermeow/data/response/messaging.go new file mode 100644 index 0000000..ee0cbb2 --- /dev/null +++ b/pkg/twittermeow/data/response/messaging.go @@ -0,0 +1,66 @@ +package response + +import ( + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/event" +) + +type InboxTimelineResponse struct { + InboxTimeline XInboxData `json:"inbox_timeline"` +} + +type ConversationDMResponse struct { + ConversationTimeline XInboxData `json:"conversation_timeline"` +} + +type InboxInitialStateResponse struct { + InboxInitialState XInboxData `json:"inbox_initial_state,omitempty"` +} +type Trusted struct { + Status types.PaginationStatus `json:"status,omitempty"` + MinEntryID string `json:"min_entry_id,omitempty"` +} +type Untrusted struct { + Status types.PaginationStatus `json:"status,omitempty"` +} +type InboxTimelines struct { + Trusted Trusted `json:"trusted,omitempty"` + Untrusted Untrusted `json:"untrusted,omitempty"` +} + +type KeyRegistryState struct { + Status types.PaginationStatus `json:"status,omitempty"` +} +type InboxInitialState struct { + LastSeenEventID string `json:"last_seen_event_id,omitempty"` + TrustedLastSeenEventID string `json:"trusted_last_seen_event_id,omitempty"` + UntrustedLastSeenEventID string `json:"untrusted_last_seen_event_id,omitempty"` + Cursor string `json:"cursor,omitempty"` + InboxTimelines InboxTimelines `json:"inbox_timelines,omitempty"` + Entries []map[event.XEventType]interface{} `json:"entries,omitempty"` + Users map[string]types.User `json:"users,omitempty"` + Conversations map[string]types.Conversation `json:"conversations,omitempty"` + KeyRegistryState KeyRegistryState `json:"key_registry_state,omitempty"` +} + +type DMMessageDeleteMutationResponse struct { + Data struct { + DmMessageHideDelete string `json:"dm_message_hide_delete,omitempty"` + } `json:"data,omitempty"` +} + +type PinConversationResponse struct { + Data struct { + AddDmConversationLabelV3 struct { + Typename string `json:"__typename,omitempty"` + LabelType string `json:"label_type,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + } `json:"add_dm_conversation_label_v3,omitempty"` + } +} + +type UnpinConversationResponse struct { + Data struct { + DmConversationLabelDelete string `json:"dm_conversation_label_delete,omitempty"` + } `json:"data,omitempty"` +} diff --git a/pkg/twittermeow/data/response/search.go b/pkg/twittermeow/data/response/search.go new file mode 100644 index 0000000..f1c8b4c --- /dev/null +++ b/pkg/twittermeow/data/response/search.go @@ -0,0 +1,16 @@ +package response + +import "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" + +type SearchResponse struct { + NumResults int `json:"num_results,omitempty"` + Users []types.User `json:"users,omitempty"` + Topics []any `json:"topics,omitempty"` + Events []any `json:"events,omitempty"` + Lists []any `json:"lists,omitempty"` + OrderedSections []any `json:"ordered_sections,omitempty"` + Oneclick []any `json:"oneclick,omitempty"` + Hashtags []any `json:"hashtags,omitempty"` + CompletedIn float32 `json:"completed_in,omitempty"` + Query string `json:"query,omitempty"` +} diff --git a/pkg/twittermeow/data/types/entities.go b/pkg/twittermeow/data/types/entities.go new file mode 100644 index 0000000..2d6b1c3 --- /dev/null +++ b/pkg/twittermeow/data/types/entities.go @@ -0,0 +1,116 @@ +package types + +type Urls struct { + URL string `json:"url,omitempty"` + ExpandedURL string `json:"expanded_url,omitempty"` + DisplayURL string `json:"display_url,omitempty"` + Indices []int `json:"indices,omitempty"` +} +type Entities struct { + Hashtags []any `json:"hashtags,omitempty"` + Symbols []any `json:"symbols,omitempty"` + UserMentions []any `json:"user_mentions,omitempty"` + Urls []Urls `json:"urls,omitempty"` +} +type OriginalInfo struct { + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` +} +type Thumb struct { + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` + Resize string `json:"resize,omitempty"` +} +type Small struct { + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` + Resize string `json:"resize,omitempty"` +} +type Large struct { + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` + Resize string `json:"resize,omitempty"` +} +type Medium struct { + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` + Resize string `json:"resize,omitempty"` +} +type Sizes struct { + Thumb Thumb `json:"thumb,omitempty"` + Small Small `json:"small,omitempty"` + Large Large `json:"large,omitempty"` + Medium Medium `json:"medium,omitempty"` +} +type Variants struct { + Bitrate int `json:"bitrate,omitempty"` + ContentType string `json:"content_type,omitempty"` + URL string `json:"url,omitempty"` +} +type VideoInfo struct { + AspectRatio []int `json:"aspect_ratio,omitempty"` + Variants []Variants `json:"variants,omitempty"` +} +type Features struct { +} +type Rgb struct { + Red int `json:"red,omitempty"` + Green int `json:"green,omitempty"` + Blue int `json:"blue,omitempty"` +} +type Palette struct { + Rgb Rgb `json:"rgb,omitempty"` + Percentage float64 `json:"percentage,omitempty"` +} +type ExtMediaColor struct { + Palette []Palette `json:"palette,omitempty"` +} +type MediaStats struct { + R string `json:"r,omitempty"` + TTL int `json:"ttl,omitempty"` +} +type Ok struct { + Palette []Palette `json:"palette,omitempty"` +} +type R struct { + Ok Ok `json:"ok,omitempty"` +} +type MediaColor struct { + R R `json:"r,omitempty"` + TTL int `json:"ttl,omitempty"` +} +type AltTextR struct { + Ok string `json:"ok,omitempty"` +} +type AltText struct { + // this is weird, it can be both string or AltTextR struct object + R interface{} `json:"r,omitempty"` + TTL int `json:"ttl,omitempty"` +} +type Ext struct { + MediaStats MediaStats `json:"mediaStats,omitempty"` + MediaColor MediaColor `json:"mediaColor,omitempty"` + AltText AltText `json:"altText,omitempty"` +} +type AnimatedGif struct { + ID int64 `json:"id,omitempty"` + IDStr string `json:"id_str,omitempty"` + Indices []int `json:"indices,omitempty"` + MediaURL string `json:"media_url,omitempty"` + MediaURLHTTPS string `json:"media_url_https,omitempty"` + URL string `json:"url,omitempty"` + DisplayURL string `json:"display_url,omitempty"` + ExpandedURL string `json:"expanded_url,omitempty"` + Type string `json:"type,omitempty"` + OriginalInfo OriginalInfo `json:"original_info,omitempty"` + Sizes Sizes `json:"sizes,omitempty"` + VideoInfo VideoInfo `json:"video_info,omitempty"` + Features Features `json:"features,omitempty"` + ExtMediaColor ExtMediaColor `json:"ext_media_color,omitempty"` + ExtAltText string `json:"ext_alt_text,omitempty"` + Ext Ext `json:"ext,omitempty"` + AudioOnly bool `json:"audio_only,omitempty"` +} +type Attachment struct { + AnimatedGif AnimatedGif `json:"animated_gif,omitempty"` +} diff --git a/pkg/twittermeow/data/types/messaging.go b/pkg/twittermeow/data/types/messaging.go new file mode 100644 index 0000000..02b49fc --- /dev/null +++ b/pkg/twittermeow/data/types/messaging.go @@ -0,0 +1,99 @@ +package types + +type MessageData struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + RecipientID string `json:"recipient_id,omitempty"` + SenderID string `json:"sender_id,omitempty"` + Text string `json:"text,omitempty"` + Entities Entities `json:"entities,omitempty"` + Attachment Attachment `json:"attachment,omitempty"` +} + +type Message struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + AffectsSort bool `json:"affects_sort,omitempty"` + RequestID string `json:"request_id,omitempty"` + ConversationID string `json:"conversation_id,omitempty"` + MessageData MessageData `json:"message_data,omitempty"` +} + +type ConversationRead struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + AffectsSort bool `json:"affects_sort,omitempty"` + ConversationID string `json:"conversation_id,omitempty"` + LastReadEventID string `json:"last_read_event_id,omitempty"` +} + +type ConversationMetadataUpdate struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + AffectsSort bool `json:"affects_sort,omitempty"` + ConversationID string `json:"conversation_id,omitempty"` +} + +type ConversationCreatedData struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + AffectsSort bool `json:"affects_sort,omitempty"` + ConversationID string `json:"conversation_id,omitempty"` + RequestID string `json:"request_id,omitempty"` +} + +type ConversationType string + +const ( + ONE_TO_ONE ConversationType = "ONE_TO_ONE" + GROUP_DM ConversationType = "GROUP_DM" +) + +type PaginationStatus string + +const ( + AT_END PaginationStatus = "AT_END" + HAS_MORE PaginationStatus = "HAS_MORE" +) + +type Conversation struct { + ConversationID string `json:"conversation_id,omitempty"` + Type ConversationType `json:"type,omitempty"` + SortEventID string `json:"sort_event_id,omitempty"` + SortTimestamp string `json:"sort_timestamp,omitempty"` + CreateTime string `json:"create_time,omitempty"` + CreatedByUserID string `json:"created_by_user_id,omitempty"` + Participants []Participants `json:"participants,omitempty"` + Nsfw bool `json:"nsfw,omitempty"` + NotificationsDisabled bool `json:"notifications_disabled,omitempty"` + MentionNotificationsDisabled bool `json:"mention_notifications_disabled,omitempty"` + LastReadEventID string `json:"last_read_event_id,omitempty"` + ReadOnly bool `json:"read_only,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Muted bool `json:"muted,omitempty"` + Status PaginationStatus `json:"status,omitempty"` + MinEntryID string `json:"min_entry_id,omitempty"` + MaxEntryID string `json:"max_entry_id,omitempty"` +} + +type Participants struct { + UserID string `json:"user_id,omitempty"` + LastReadEventID string `json:"last_read_event_id,omitempty"` + IsAdmin bool `json:"is_admin,omitempty"` + JoinTime string `json:"join_time,omitempty"` + JoinConversationEventId string `json:"join_conversation_event_id,omitempty"` +} + +type MessageDeleted struct { + ID string `json:"id,omitempty"` + Time string `json:"time,omitempty"` + AffectsSort bool `json:"affects_sort,omitempty"` + RequestID string `json:"request_id,omitempty"` + ConversationID string `json:"conversation_id,omitempty"` + Messages []MessagesDeleted `json:"messages,omitempty"` +} + +type MessagesDeleted struct { + MessageID string `json:"message_id,omitempty"` + MessageCreateEventID string `json:"message_create_event_id,omitempty"` +} diff --git a/pkg/twittermeow/data/types/user.go b/pkg/twittermeow/data/types/user.go new file mode 100644 index 0000000..5e9ea69 --- /dev/null +++ b/pkg/twittermeow/data/types/user.go @@ -0,0 +1,49 @@ +package types + +type User struct { + ID int64 `json:"id,omitempty"` + IDStr string `json:"id_str,omitempty"` + Name string `json:"name,omitempty"` + ScreenName string `json:"screen_name,omitempty"` + ProfileImageURL string `json:"profile_image_url,omitempty"` + ProfileImageURLHTTPS string `json:"profile_image_url_https,omitempty"` + Following bool `json:"following,omitempty"` + FollowRequestSent bool `json:"follow_request_sent,omitempty"` + Description string `json:"description,omitempty"` + Entities Entities `json:"entities,omitempty"` + Verified bool `json:"verified,omitempty"` + IsBlueVerified bool `json:"is_blue_verified,omitempty"` + ExtIsBlueVerified bool `json:"ext_is_blue_verified,omitempty"` + Protected bool `json:"protected,omitempty"` + IsProtected bool `json:"is_protected,omitempty"` + Blocking bool `json:"blocking,omitempty"` + IsBlocked bool `json:"is_blocked,omitempty"` + IsSecretDmAble bool `json:"is_secret_dm_able,omitempty"` + IsDmAble bool `json:"is_dm_able,omitempty"` + SubscribedBy bool `json:"subscribed_by,omitempty"` + CanMediaTag bool `json:"can_media_tag,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Location string `json:"location,omitempty"` + FriendsCount int `json:"friends_count,omitempty"` + SocialProof int `json:"social_proof,omitempty"` + RoundedScore int `json:"rounded_score,omitempty"` + FollowersCount int `json:"followers_count,omitempty"` + ConnectingUserCount int `json:"connecting_user_count,omitempty"` + ConnectingUserIds []any `json:"connecting_user_ids,omitempty"` + SocialProofsOrdered []any `json:"social_proofs_ordered,omitempty"` + Tokens []any `json:"tokens,omitempty"` + Inline bool `json:"inline,omitempty"` +} + +type UserEntities struct { + URL URL `json:"url,omitempty"` + Description Description `json:"description,omitempty"` +} + +type URL struct { + Urls []Urls `json:"urls,omitempty"` +} + +type Description struct { + Urls []Urls `json:"urls,omitempty"` +} diff --git a/pkg/twittermeow/debug/logger.go b/pkg/twittermeow/debug/logger.go new file mode 100644 index 0000000..ae34145 --- /dev/null +++ b/pkg/twittermeow/debug/logger.go @@ -0,0 +1,56 @@ +package debug + +import ( + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/mattn/go-colorable" + zerolog "github.com/rs/zerolog" +) + +var colors = map[string]string{ + "text": "\x1b[38;5;6m%s\x1b[0m", + "debug": "\x1b[32mDEBUG\x1b[0m", + "gray": "\x1b[38;5;8m%s\x1b[0m", + "info": "\x1b[38;5;111mINFO\x1b[0m", + "error": "\x1b[38;5;204mERROR\x1b[0m", + "fatal": "\x1b[38;5;52mFATAL\x1b[0m", +} + +var output = zerolog.ConsoleWriter{ + Out: colorable.NewColorableStdout(), + TimeFormat: time.ANSIC, + FormatLevel: func(i interface{}) string { + name := fmt.Sprintf("%s", i) + coloredName := colors[name] + return coloredName + }, + FormatMessage: func(i interface{}) string { + coloredMsg := fmt.Sprintf(colors["text"], i) + return coloredMsg + }, + FormatFieldName: func(i interface{}) string { + name := fmt.Sprintf("%s", i) + return fmt.Sprintf(colors["gray"], name+"=") + }, + FormatFieldValue: func(i interface{}) string { + return fmt.Sprintf("%s", i) + }, + NoColor: false, +} + +func NewLogger() zerolog.Logger { + return zerolog.New(output).With().Timestamp().Logger() +} + +func BeautifyHex(data []byte) string { + hexStr := hex.EncodeToString(data) + result := "" + for i := 0; i < len(hexStr); i += 2 { + result += hexStr[i:i+2] + " " + } + + return strings.TrimRight(result, " ") +} diff --git a/pkg/twittermeow/errors.go b/pkg/twittermeow/errors.go new file mode 100644 index 0000000..af2309f --- /dev/null +++ b/pkg/twittermeow/errors.go @@ -0,0 +1,16 @@ +package twittermeow + +import "errors" + +var ( + // Connection errors + ErrConnectPleaseSetEventHandler = errors.New("please set event handler in client before connecting") + ErrNotAuthenticatedYet = errors.New("client has not been authenticated yet") + + // Polling errors + ErrAlreadyPollingUpdates = errors.New("client is already polling for user updates") + ErrNotPollingUpdates = errors.New("client is not polling for user updates") + + // Api errors + ErrFailedMarkConversationRead = errors.New("failed to mark conversation as read") +) diff --git a/pkg/twittermeow/event/event.go b/pkg/twittermeow/event/event.go new file mode 100644 index 0000000..eff5727 --- /dev/null +++ b/pkg/twittermeow/event/event.go @@ -0,0 +1,62 @@ +package event + +import ( + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" +) + +type XEventType string + +const ( + XMessageEvent XEventType = "message" + XMessageDeleteEvent XEventType = "message_delete" + XConversationReadEvent XEventType = "conversation_read" + XConversationMetadataUpdateEvent XEventType = "conversation_metadata_update" + XConversationCreateEvent XEventType = "conversation_create" + XDisableNotificationsEvent XEventType = "disable_notifications" +) + +type XEventMessage struct { + Conversation types.Conversation + Sender types.User + Recipient types.User + MessageID string + Text string + CreatedAt time.Time + AffectsSort bool + Entities types.Entities + Attachment types.Attachment +} + +type XEventConversationRead struct { + EventID string + Conversation types.Conversation + ReadAt time.Time + AffectsSort bool + LastReadEventID string +} + +type XEventConversationCreated struct { + EventID string + Conversation types.Conversation + CreatedAt time.Time + AffectsSort bool + RequestID string +} + +type XEventConversationMetadataUpdate struct { + Conversation types.Conversation + EventID string + UpdatedAt time.Time + AffectsSort bool +} + +type XEventMessageDeleted struct { + Conversation types.Conversation + EventID string + RequestID string + DeletedAt time.Time + AffectsSort bool + Messages []types.MessagesDeleted +} diff --git a/pkg/twittermeow/events.go b/pkg/twittermeow/events.go new file mode 100644 index 0000000..16295e3 --- /dev/null +++ b/pkg/twittermeow/events.go @@ -0,0 +1,18 @@ +package twittermeow + +import ( + "log" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" +) + +func (c *Client) processEventEntries(resp *response.GetDMUserUpdatesResponse) { + entries, err := resp.UserEvents.ToEventEntries() + if err != nil { + log.Fatal(err) // send event handler error + } + + for _, entry := range entries { + c.eventHandler(entry) + } +} diff --git a/pkg/twittermeow/headers.go b/pkg/twittermeow/headers.go new file mode 100644 index 0000000..d3094de --- /dev/null +++ b/pkg/twittermeow/headers.go @@ -0,0 +1,101 @@ +package twittermeow + +import ( + "net/http" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/cookies" +) + +const BrowserName = "Chrome" +const ChromeVersion = "118" +const ChromeVersionFull = ChromeVersion + ".0.5993.89" +const UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/" + ChromeVersion + ".0.0.0 Safari/537.36" +const SecCHUserAgent = `"Chromium";v="` + ChromeVersion + `", "Google Chrome";v="` + ChromeVersion + `", "Not-A.Brand";v="99"` +const SecCHFullVersionList = `"Chromium";v="` + ChromeVersionFull + `", "Google Chrome";v="` + ChromeVersionFull + `", "Not-A.Brand";v="99.0.0.0"` +const OSName = "Linux" +const OSVersion = "6.5.0" +const SecCHPlatform = `"` + OSName + `"` +const SecCHPlatformVersion = `"` + OSVersion + `"` +const SecCHMobile = "?0" +const SecCHModel = "" +const SecCHPrefersColorScheme = "light" + +var defaultConstantHeaders = http.Header{ + "accept": []string{"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"}, + "accept-language": []string{"en-US,en;q=0.9"}, + "user-agent": []string{UserAgent}, + "sec-ch-ua": []string{SecCHUserAgent}, + "sec-ch-ua-platform": []string{SecCHPlatform}, + "sec-ch-prefers-color-scheme": []string{SecCHPrefersColorScheme}, + "sec-ch-ua-full-version-list": []string{SecCHFullVersionList}, + "sec-ch-ua-mobile": []string{SecCHMobile}, + // "sec-ch-ua-model": []string{SecCHModel}, + // "sec-ch-ua-platform-version": []string{SecCHPlatformVersion}, +} + +type HeaderOpts struct { + WithAuthBearer bool + WithCookies bool + WithXGuestToken bool + WithXTwitterHeaders bool + WithXCsrfToken bool + WithXClientUUID bool + Referer string + Origin string + Extra map[string]string +} + +func (c *Client) buildHeaders(opts HeaderOpts) http.Header { + if opts.Extra == nil { + opts.Extra = make(map[string]string, 0) + } + + headers := defaultConstantHeaders.Clone() + if opts.WithCookies { + opts.Extra["cookie"] = c.cookies.String() + } + + if opts.WithAuthBearer { + authTokens := c.session.GetAuthTokens() + // check if client is authenticated here + var bearerToken string + if c.isAuthenticated() { + bearerToken = authTokens.authenticated + } else { + bearerToken = authTokens.notAuthenticated + } + opts.Extra["authorization"] = bearerToken + } + + if opts.WithXGuestToken { + opts.Extra["x-guest-token"] = c.cookies.Get(cookies.XGuestToken) + } + + if opts.WithXClientUUID { + opts.Extra["x-client-uuid"] = c.session.clientUUID + } + + if opts.WithXTwitterHeaders { + opts.Extra["x-twitter-active-user"] = "yes" + opts.Extra["x-twitter-client-language"] = "en" + } + + if opts.WithXCsrfToken { + opts.Extra["x-csrf-token"] = c.cookies.Get(cookies.XCt0) + opts.Extra["x-twitter-auth-type"] = "OAuth2Session" + } + + if opts.Origin != "" { + opts.Extra["origin"] = opts.Origin + } + + if opts.Referer != "" { + opts.Extra["referer"] = opts.Referer + } + + for k, v := range opts.Extra { + headers.Set(k, v) + } + + return headers +} diff --git a/pkg/twittermeow/http.go b/pkg/twittermeow/http.go new file mode 100644 index 0000000..1f2a677 --- /dev/null +++ b/pkg/twittermeow/http.go @@ -0,0 +1,114 @@ +package twittermeow + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" +) + +const MaxHTTPRetries = 5 + +var ( + ErrRedirectAttempted = errors.New("redirect attempted") + ErrTokenInvalidated = errors.New("access token is no longer valid") + ErrChallengeRequired = errors.New("challenge required") + ErrConsentRequired = errors.New("consent required") + ErrAccountSuspended = errors.New("account suspended") + ErrRequestFailed = errors.New("failed to send request") + ErrResponseReadFailed = errors.New("failed to read response body") + ErrMaxRetriesReached = errors.New("maximum retries reached") +) + +func isPermanentRequestError(err error) bool { + return errors.Is(err, ErrTokenInvalidated) || + errors.Is(err, ErrChallengeRequired) || + errors.Is(err, ErrConsentRequired) || + errors.Is(err, ErrAccountSuspended) +} + +func (c *Client) MakeRequest(url string, method string, headers http.Header, payload []byte, contentType types.ContentType) (*http.Response, []byte, error) { + var attempts int + for { + attempts++ + start := time.Now() + resp, respDat, err := c.makeRequestDirect(url, method, headers, payload, contentType) + dur := time.Since(start) + if err == nil { + c.Logger.Debug(). + Str("url", url). + Str("method", method). + Dur("duration", dur). + Msg("Request successful") + return resp, respDat, nil + } else if attempts > MaxHTTPRetries { + c.Logger.Err(err). + Str("url", url). + Str("method", method). + Dur("duration", dur). + Msg("Request failed, giving up") + return nil, nil, fmt.Errorf("%w: %w", ErrMaxRetriesReached, err) + } else if isPermanentRequestError(err) { + c.Logger.Err(err). + Str("url", url). + Str("method", method). + Dur("duration", dur). + Msg("Request failed, cannot be retried") + return nil, nil, err + } else if errors.Is(err, ErrRedirectAttempted) { + location := resp.Header.Get("Location") + c.Logger.Err(err). + Str("url", url). + Str("location", location). + Str("method", method). + Dur("duration", dur). + Msg("Redirect attempted") + return resp, nil, err + } + c.Logger.Err(err). + Str("url", url). + Str("method", method). + Dur("duration", dur). + Msg("Request failed, retrying") + time.Sleep(time.Duration(attempts) * 3 * time.Second) + } +} + +func (c *Client) makeRequestDirect(url string, method string, headers http.Header, payload []byte, contentType types.ContentType) (*http.Response, []byte, error) { + newRequest, err := http.NewRequest(method, url, bytes.NewBuffer(payload)) + if err != nil { + return nil, nil, err + } + + if contentType != types.NONE { + headers.Set("content-type", string(contentType)) + } + + newRequest.Header = headers + + response, err := c.http.Do(newRequest) + defer func() { + if response != nil && response.Body != nil { + _ = response.Body.Close() + } + }() + if err != nil { + if errors.Is(err, ErrRedirectAttempted) { + return response, nil, err + } + c.Logger.Warn().Str("error", err.Error()).Msg("Http request error") + // c.UpdateProxy(fmt.Sprintf("http request error: %v", err.Error())) + return nil, nil, fmt.Errorf("%w: %w", ErrRequestFailed, err) + } + + responseBody, err := io.ReadAll(response.Body) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", ErrResponseReadFailed, err) + } + + return response, responseBody, nil +} diff --git a/pkg/twittermeow/jot_client.go b/pkg/twittermeow/jot_client.go new file mode 100644 index 0000000..f930c44 --- /dev/null +++ b/pkg/twittermeow/jot_client.go @@ -0,0 +1,73 @@ +package twittermeow + +import ( + "encoding/json" + "fmt" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/crypto" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" +) + +type JotClient struct { + client *Client +} + +func (c *Client) newJotClient() *JotClient { + return &JotClient{ + client: c, + } +} + +func (jc *JotClient) sendClientLoggingEvent(category payload.JotLoggingCategory, debug bool, body []interface{}) error { + logPayloadBytes, err := json.Marshal(body) + if err != nil { + return err + } + + clientLogPayload := &payload.JotClientEventPayload{ + Category: category, + Debug: debug, + Log: string(logPayloadBytes), + } + + clientLogPayloadBytes, err := clientLogPayload.Encode() + if err != nil { + return err + } + + clientTransactionId, err := crypto.SignTransaction(jc.client.session.verificationToken, endpoints.JOT_CLIENT_EVENT_URL, "POST") + if err != nil { + return err + } + + extraHeaders := map[string]string{ + "accept": "*/*", + "sec-fetch-site": "same-site", + "sec-fetch-mode": "cors", + "sec-fetch-dest": "empty", + "x-client-transaction-id": clientTransactionId, + } + + headerOpts := HeaderOpts{ + WithAuthBearer: true, + WithCookies: true, + WithXGuestToken: true, + WithXTwitterHeaders: true, + Origin: endpoints.BASE_URL, + Referer: endpoints.BASE_URL + "/", + Extra: extraHeaders, + } + + clientLogResponse, _, err := jc.client.MakeRequest(endpoints.JOT_CLIENT_EVENT_URL, "POST", jc.client.buildHeaders(headerOpts), clientLogPayloadBytes, types.FORM) + if err != nil { + return err + } + + if clientLogResponse.StatusCode > 204 { + return fmt.Errorf("failed to send jot client event, status code: %d", clientLogResponse.StatusCode) + } + + return nil +} diff --git a/pkg/twittermeow/media.go b/pkg/twittermeow/media.go new file mode 100644 index 0000000..1fc2069 --- /dev/null +++ b/pkg/twittermeow/media.go @@ -0,0 +1,181 @@ +package twittermeow + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "log" + "mime/multipart" + "net/http" + "net/textproto" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" +) + +func (c *Client) UploadMedia(params *payload.UploadMediaQuery, mediaBytes []byte) (*response.FinalizedUploadMediaResponse, error) { + params.Command = "INIT" + if mediaBytes != nil { + params.TotalBytes = len(mediaBytes) + } + + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + + var finalizedMediaResultBytes []byte + + url := fmt.Sprintf("%s?%s", endpoints.UPLOAD_MEDIA_URL, string(encodedQuery)) + headerOpts := HeaderOpts{ + WithAuthBearer: true, + WithXCsrfToken: true, + WithCookies: true, + Origin: endpoints.BASE_URL, + Referer: endpoints.BASE_URL + "/", + } + headers := c.buildHeaders(headerOpts) + + _, respBody, err := c.MakeRequest(url, "POST", headers, nil, types.NONE) + if err != nil { + return nil, err + } + + initUploadResponse := &response.InitUploadMediaResponse{} + err = json.Unmarshal(respBody, initUploadResponse) + if err != nil { + return nil, err + } + + if mediaBytes != nil { + appendMediaPayload, contentType, err := c.newMediaAppendPayload(mediaBytes) + if err != nil { + return nil, err + } + headers.Add("content-type", contentType) + + url = fmt.Sprintf("%s?command=APPEND&media_id=%s&segment_index=0", endpoints.UPLOAD_MEDIA_URL, initUploadResponse.MediaIDString) + resp, respBody, err := c.MakeRequest(url, "POST", headers, appendMediaPayload, types.NONE) + if err != nil { + return nil, err + } + + if resp.StatusCode > 204 { + return nil, fmt.Errorf("failed to append media bytes for media with id %s (status_code=%d, response_body=%s)", initUploadResponse.MediaIDString, resp.StatusCode, string(respBody)) + } + + var originalMd5 string + if params.MediaCategory == payload.MEDIA_CATEGORY_DM_IMAGE { + md5Hash := md5.Sum(mediaBytes) + originalMd5 = hex.EncodeToString(md5Hash[:]) + } + + finalizeMediaQuery := &payload.UploadMediaQuery{ + Command: "FINALIZE", + MediaID: initUploadResponse.MediaIDString, + OriginalMd5: originalMd5, + } + + encodedQuery, err = finalizeMediaQuery.Encode() + if err != nil { + return nil, err + } + + url = fmt.Sprintf("%s?%s", endpoints.UPLOAD_MEDIA_URL, string(encodedQuery)) + log.Println("url", url) + headers.Del("content-type") + resp, respBody, err = c.MakeRequest(url, "POST", headers, nil, types.NONE) + if err != nil { + return nil, err + } + + if resp.StatusCode > 204 { + return nil, fmt.Errorf("failed to finalize media with id %s (status_code=%d, response_body=%s)", initUploadResponse.MediaIDString, resp.StatusCode, string(respBody)) + } + + finalizedMediaResultBytes = respBody + } else { + _, finalizedMediaResultBytes, err = c.GetMediaUploadStatus(initUploadResponse.MediaIDString, headers) + if err != nil { + return nil, err + } + } + + finalizedMediaResult := &response.FinalizedUploadMediaResponse{} + err = json.Unmarshal(finalizedMediaResultBytes, finalizedMediaResult) + if err != nil { + return nil, err + } + + if finalizedMediaResult.ProcessingInfo.State == response.PROCESSING_STATE_PENDING || finalizedMediaResult.ProcessingInfo.State == response.PROCESSING_STATE_IN_PROGRESS { + // might need to check for error processing states here, I have not encountered any though so I wouldn't know what they look like/are + for finalizedMediaResult.ProcessingInfo.State != response.PROCESSING_STATE_SUCCEEDED { + finalizedMediaResult, _, err = c.GetMediaUploadStatus(finalizedMediaResult.MediaIDString, headers) + if err != nil { + return nil, err + } + c.Logger.Debug(). + Int("progress_percent", finalizedMediaResult.ProcessingInfo.ProgressPercent). + Int("status_check_interval_seconds", finalizedMediaResult.ProcessingInfo.CheckAfterSecs). + Str("media_id", finalizedMediaResult.MediaIDString). + Str("state", string(finalizedMediaResult.ProcessingInfo.State)). + Msg("Waiting for X to finish processing our media upload...") + time.Sleep(time.Second * time.Duration(finalizedMediaResult.ProcessingInfo.CheckAfterSecs)) + } + } + + return finalizedMediaResult, nil +} + +func (c *Client) GetMediaUploadStatus(mediaId string, h http.Header) (*response.FinalizedUploadMediaResponse, []byte, error) { + url := fmt.Sprintf("%s?command=STATUS&media_id=%s", endpoints.UPLOAD_MEDIA_URL, mediaId) + resp, respBody, err := c.MakeRequest(url, "GET", h, nil, types.NONE) + if err != nil { + return nil, nil, err + } + + if resp.StatusCode > 204 { + return nil, nil, fmt.Errorf("failed to get status of uploaded media with id %s (status_code=%d, response_body=%s)", mediaId, resp.StatusCode, string(respBody)) + } + + mediaStatusResult := &response.FinalizedUploadMediaResponse{} + return mediaStatusResult, respBody, json.Unmarshal(respBody, mediaStatusResult) +} + +func (c *Client) newMediaAppendPayload(mediaBytes []byte) ([]byte, string, error) { + var appendMediaPayload bytes.Buffer + writer := multipart.NewWriter(&appendMediaPayload) + + err := writer.SetBoundary("----WebKitFormBoundary" + methods.RandStr(16)) + if err != nil { + return nil, "", fmt.Errorf("failed to set boundary (%s)", err.Error()) + } + + partHeader := textproto.MIMEHeader{ + "Content-Disposition": []string{`form-data; name="media"; filename="blob"`}, + "Content-Type": []string{"application/octet-stream"}, + } + + mediaPart, err := writer.CreatePart(partHeader) + if err != nil { + return nil, "", fmt.Errorf("failed to create multipart writer (%s)", err.Error()) + } + + _, err = mediaPart.Write(mediaBytes) + if err != nil { + return nil, "", fmt.Errorf("failed to write data to multipart section (%s)", err.Error()) + } + + err = writer.Close() + if err != nil { + return nil, "", fmt.Errorf("failed to close multipart writer (%s)", err.Error()) + } + + return appendMediaPayload.Bytes(), writer.FormDataContentType(), nil +} diff --git a/pkg/twittermeow/messaging.go b/pkg/twittermeow/messaging.go new file mode 100644 index 0000000..e93fc7f --- /dev/null +++ b/pkg/twittermeow/messaging.go @@ -0,0 +1,282 @@ +package twittermeow + +import ( + "encoding/json" + "fmt" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" + + "github.com/google/uuid" +) + +func (c *Client) GetInitialInboxState(params *payload.DmRequestQuery) (*response.InboxInitialStateResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.INBOX_INITIAL_STATE_URL, string(encodedQuery)) + + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.InboxInitialStateResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) GetDMUserUpdates(params *payload.DmRequestQuery) (*response.GetDMUserUpdatesResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.DM_USER_UPDATES_URL, string(encodedQuery)) + + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.GetDMUserUpdatesResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) MarkConversationRead(params *payload.MarkConversationReadQuery) error { + encodedQueryBody, err := params.Encode() + if err != nil { + return err + } + + url := fmt.Sprintf(endpoints.CONVERSATION_MARK_READ_URL, params.ConversationID) + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: encodedQueryBody, + ContentType: types.FORM, + } + resp, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return err + } + + if resp.StatusCode > 204 { + c.Logger.Warn().Any("response_body", string(respBody)).Any("status_code", resp.StatusCode).Any("params", params).Msg("Failed to mark conversation as read") + return ErrFailedMarkConversationRead + } + + return nil +} + +func (c *Client) FetchConversationContext(conversationId string, params *payload.DmRequestQuery, context payload.ContextInfo) (*response.ConversationDMResponse, error) { + params.Context = context + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", fmt.Sprintf(endpoints.CONVERSATION_FETCH_MESSAGES, conversationId), string(encodedQuery)) + + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.ConversationDMResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) FetchTrustedThreads(params *payload.DmRequestQuery) (*response.InboxTimelineResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.TRUSTED_INBOX_TIMELINE_URL, string(encodedQuery)) + + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.InboxTimelineResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) SendDirectMessage(payload *payload.SendDirectMessagePayload) (*response.XInboxData, error) { + if payload.RequestID == "" { + payload.RequestID = uuid.NewString() + } + + jsonBody, err := payload.Encode() + if err != nil { + return nil, err + } + + url := endpoints.SEND_DM_URL + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: jsonBody, + Referer: fmt.Sprintf("%s/%s", endpoints.BASE_MESSAGES_URL, payload.ConversationID), + Origin: endpoints.BASE_URL, + ContentType: types.JSON, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.XInboxData{} + return &data, json.Unmarshal(respBody, &data) +} + +// keep in mind this only deletes the message for you +func (c *Client) DeleteMessage(variables *payload.DMMessageDeleteMutationVariables) (*response.DMMessageDeleteMutationResponse, error) { + if variables.RequestID == "" { + variables.RequestID = uuid.NewString() + } + + payload := &payload.GraphQLPayload{ + Variables: variables, + QueryID: "BJ6DtxA2llfjnRoRjaiIiw", + } + + jsonBody, err := payload.Encode() + if err != nil { + return nil, err + } + + url := endpoints.GRAPHQL_MESSAGE_DELETION_MUTATION + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: jsonBody, + Origin: endpoints.BASE_URL, + ContentType: types.JSON, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.DMMessageDeleteMutationResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) DeleteConversation(conversationId string, payload *payload.DmRequestQuery) error { + encodedQueryBody, err := payload.Encode() + if err != nil { + return err + } + + url := fmt.Sprintf(endpoints.DELETE_CONVERSATION_URL, conversationId) + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: encodedQueryBody, + Referer: endpoints.BASE_MESSAGES_URL, + Origin: endpoints.BASE_URL, + ContentType: types.FORM, + } + resp, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return err + } + + if resp.StatusCode > 204 { + return fmt.Errorf("failed to delete conversation by id %s (status_code=%d, response_body=%s)", conversationId, resp.StatusCode, string(respBody)) + } + + return nil +} + +func (c *Client) PinConversation(conversationId string) (*response.PinConversationResponse, error) { + graphQlPayload := payload.GraphQLPayload{ + Variables: payload.PinAndUnpinConversationVariables{ + ConversationID: conversationId, + Label: payload.LABEL_TYPE_PINNED, + }, + QueryID: "o0aymgGiJY-53Y52YSUGVA", + } + + jsonBody, err := graphQlPayload.Encode() + if err != nil { + return nil, err + } + + url := endpoints.PIN_CONVERSATION_URL + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: jsonBody, + Referer: endpoints.BASE_MESSAGES_URL, + Origin: endpoints.BASE_URL, + ContentType: types.JSON, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.PinConversationResponse{} + return &data, json.Unmarshal(respBody, &data) +} + +func (c *Client) UnpinConversation(conversationId string) (*response.UnpinConversationResponse, error) { + graphQlPayload := payload.GraphQLPayload{ + Variables: payload.PinAndUnpinConversationVariables{ + ConversationID: conversationId, + LabelType: payload.LABEL_TYPE_PINNED, + }, + QueryID: "_TQxP2Rb0expwVP9ktGrTQ", + } + + jsonBody, err := graphQlPayload.Encode() + if err != nil { + return nil, err + } + + url := endpoints.UNPIN_CONVERSATION_URL + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "POST", + WithClientUUID: true, + Body: jsonBody, + Referer: endpoints.BASE_MESSAGES_URL, + Origin: endpoints.BASE_URL, + ContentType: types.JSON, + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.UnpinConversationResponse{} + return &data, json.Unmarshal(respBody, &data) +} diff --git a/pkg/twittermeow/methods/html.go b/pkg/twittermeow/methods/html.go new file mode 100644 index 0000000..f864b44 --- /dev/null +++ b/pkg/twittermeow/methods/html.go @@ -0,0 +1,70 @@ +package methods + +import ( + "regexp" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" +) + +var ( + metaTagRegex = regexp.MustCompile(``) + migrateFormDataRegex = regexp.MustCompile(`]* action="([^"]+)"[^>]*>[\s\S]*?]* name="tok" value="([^"]+)"[^>]*>[\s\S]*?]* name="data" value="([^"]+)"[^>]*>`) + mainScriptUrlRegex = regexp.MustCompile(`https:\/\/(?:[A-Za-z0-9.-]+)\/responsive-web\/client-web\/main\.[0-9A-Za-z]+\.js`) + bearerTokenRegex = regexp.MustCompile(`(Bearer\s[A-Za-z0-9%]+)`) + guestTokenRegex = regexp.MustCompile(`gt=([0-9]+)`) + verificationTokenRegex = regexp.MustCompile(`meta name="twitter-site-verification" content="([^"]+)"`) + countryCodeRegex = regexp.MustCompile(`"country":\s*"([A-Z]{2})"`) +) + +func ParseMigrateURL(html string) (string, bool) { + match := metaTagRegex.FindStringSubmatch(html) + if len(match) > 1 { + return match[1], true + } + return "", false +} + +func ParseMigrateRequestData(html string) (string, *payload.MigrationRequestPayload) { + match := migrateFormDataRegex.FindStringSubmatch(html) + if len(match) < 4 { + return "", nil + } + + return match[1], &payload.MigrationRequestPayload{Tok: match[2], Data: match[3]} +} + +func ParseMainScriptURL(html string) string { + match := mainScriptUrlRegex.FindStringSubmatch(html) + if len(match) < 1 { + return "" + } + return match[0] +} + +func ParseBearerTokens(html string) []string { + return bearerTokenRegex.FindAllString(html, -1) +} + +func ParseGuestToken(html string) string { + match := guestTokenRegex.FindStringSubmatch(html) + if len(match) < 1 { + return "" + } + return match[1] +} + +func ParseVerificationToken(html string) string { + match := verificationTokenRegex.FindStringSubmatch(html) + if len(match) < 1 { + return "" + } + return match[1] +} + +func ParseCountry(html string) string { + match := countryCodeRegex.FindStringSubmatch(html) + if len(match) < 2 { + return "" + } + return match[1] +} diff --git a/pkg/twittermeow/methods/methods.go b/pkg/twittermeow/methods/methods.go new file mode 100644 index 0000000..90acffc --- /dev/null +++ b/pkg/twittermeow/methods/methods.go @@ -0,0 +1,79 @@ +package methods + +import ( + "math/rand" + "sort" + "strconv" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/types" +) + +var Charset = []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890") + +// retrieved from main page resp, its a 2 year old timestamp I don't think this is changing +const fetchedTime = 1661971138705 + +func GenerateEventValue() int64 { + ts := time.Now().UnixNano() / int64(time.Millisecond) + return ts - fetchedTime +} + +func GetTimestampMS() int64 { + return time.Now().UnixNano() / int64(time.Millisecond) +} + +func UnixStringMilliToTime(input string) (time.Time, error) { + secs, err := strconv.ParseInt(input, 10, 64) + if err != nil { + return time.Time{}, err + } + return time.UnixMilli(secs), nil +} + +func SortConversationsByTimestamp(conversations map[string]types.Conversation) []types.Conversation { + var conversationSlice []types.Conversation + for _, conversation := range conversations { + conversationSlice = append(conversationSlice, conversation) + } + + sort.Slice(conversationSlice, func(j, i int) bool { + timeJ, errJ := strconv.ParseInt(conversationSlice[j].SortTimestamp, 10, 64) + timeI, errI := strconv.ParseInt(conversationSlice[i].SortTimestamp, 10, 64) + + if errI != nil || errJ != nil { + return errI == nil + } + + return timeI < timeJ + }) + + return conversationSlice +} + +func SortMessagesByTime(messages []types.Message) { + sort.Slice(messages, func(j, i int) bool { + timeJ, errJ := strconv.ParseInt(messages[j].Time, 10, 64) + timeI, errI := strconv.ParseInt(messages[i].Time, 10, 64) + + if errI != nil || errJ != nil { + return errI == nil + } + + return timeI < timeJ + }) +} + +func MergeMaps[K comparable, V any](dst, src map[K]V) { + for key, value := range src { + dst[key] = value + } +} + +func RandStr(length int) string { + b := make([]rune, length) + for i := range b { + b[i] = Charset[rand.Intn(len(Charset))] + } + return string(b) +} diff --git a/pkg/twittermeow/onboarding_client.go b/pkg/twittermeow/onboarding_client.go new file mode 100644 index 0000000..4721070 --- /dev/null +++ b/pkg/twittermeow/onboarding_client.go @@ -0,0 +1,27 @@ +package twittermeow + +import ( + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" +) + +type OnboardingClient struct { + client *Client + flowToken string + currentTasks *types.TaskResponse +} + +func (c *Client) newOnboardingClient() *OnboardingClient { + return &OnboardingClient{ + client: c, + currentTasks: &types.TaskResponse{}, + } +} + +func (o *OnboardingClient) SetFlowToken(flowToken string) *OnboardingClient { + o.flowToken = flowToken + return o +} + +func (o *OnboardingClient) SetCurrentTasks(tasks *types.TaskResponse) { + o.currentTasks = tasks +} diff --git a/pkg/twittermeow/polling.go b/pkg/twittermeow/polling.go new file mode 100644 index 0000000..a1112cc --- /dev/null +++ b/pkg/twittermeow/polling.go @@ -0,0 +1,78 @@ +package twittermeow + +import ( + "log" + "time" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" +) + +var defaultPollingInterval = 10 * time.Second + +type PollingClient struct { + client *Client + interval *time.Duration + ticker *time.Ticker + currentCursor string +} + +// interval is the delay inbetween checking for new updates +// default interval will be 10s +func (c *Client) newPollingClient(interval *time.Duration) *PollingClient { + if interval == nil { + interval = &defaultPollingInterval + } + return &PollingClient{ + client: c, + interval: interval, + } +} + +func (pc *PollingClient) startPolling() error { + if pc.ticker != nil { + return ErrAlreadyPollingUpdates + } + + pc.ticker = time.NewTicker(*pc.interval) + go pc.startListening() + + return nil +} + +func (pc *PollingClient) startListening() { + userUpdatesQuery := (&payload.DmRequestQuery{}).Default() + for range pc.ticker.C { + if pc.currentCursor != "" { + userUpdatesQuery.Cursor = pc.currentCursor + } + + userUpdatesResponse, err := pc.client.GetDMUserUpdates(userUpdatesQuery) + if err != nil { + log.Fatal(err) + } + + userEvents := userUpdatesResponse.UserEvents + if len(userEvents.Entries) > 0 { + pc.client.processEventEntries(userUpdatesResponse) + } + + // pc.client.logger.Info().Any("user_events", userUpdatesResponse.UserEvents).Any("inbox_initial_state", userUpdatesResponse.InboxInitialState).Msg("Got polling update response") + + pc.SetCurrentCursor(userUpdatesResponse.UserEvents.Cursor) + } +} + +func (pc *PollingClient) SetCurrentCursor(cursor string) { + pc.currentCursor = cursor +} + +func (pc *PollingClient) stopPolling() error { + if pc.ticker == nil { + return ErrNotPollingUpdates + } + + pc.ticker.Stop() + pc.ticker = nil + + return nil +} diff --git a/pkg/twittermeow/search.go b/pkg/twittermeow/search.go new file mode 100644 index 0000000..0b27727 --- /dev/null +++ b/pkg/twittermeow/search.go @@ -0,0 +1,32 @@ +package twittermeow + +import ( + "encoding/json" + "fmt" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" +) + +func (c *Client) Search(params payload.SearchQuery) (*response.SearchResponse, error) { + encodedQuery, err := params.Encode() + if err != nil { + return nil, err + } + url := fmt.Sprintf("%s?%s", endpoints.SEARCH_TYPEAHEAD_URL, string(encodedQuery)) + + apiRequestOpts := apiRequestOpts{ + Url: url, + Method: "GET", + WithClientUUID: true, + Referer: endpoints.BASE_MESSAGES_URL + "/compose", + } + _, respBody, err := c.makeAPIRequest(apiRequestOpts) + if err != nil { + return nil, err + } + + data := response.SearchResponse{} + return &data, json.Unmarshal(respBody, &data) +} diff --git a/pkg/twittermeow/session_loader.go b/pkg/twittermeow/session_loader.go new file mode 100644 index 0000000..d27cec1 --- /dev/null +++ b/pkg/twittermeow/session_loader.go @@ -0,0 +1,271 @@ +package twittermeow + +import ( + "errors" + "fmt" + neturl "net/url" + + "go.mau.fi/mautrix-twitter/pkg/twittermeow/cookies" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/endpoints" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/payload" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/data/response" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/methods" + "go.mau.fi/mautrix-twitter/pkg/twittermeow/types" + + "github.com/google/go-querystring/query" + "github.com/google/uuid" +) + +var ( + errCookieGuestIDNotFound = errors.New("failed to retrieve and set 'guest_id' cookie from Set-Cookie response headers") +) + +type SessionAuthTokens struct { + authenticated string + notAuthenticated string +} + +type SessionLoader struct { + client *Client + currentUser *response.AccountSettingsResponse + verificationToken string + country string + clientUUID string + authTokens *SessionAuthTokens +} + +func (c *Client) newSessionLoader() *SessionLoader { + return &SessionLoader{ + client: c, + clientUUID: uuid.NewString(), + authTokens: &SessionAuthTokens{}, + } +} + +func (s *SessionLoader) SetCurrentUser(data *response.AccountSettingsResponse) { + s.currentUser = data +} + +func (s *SessionLoader) GetCurrentUser() *response.AccountSettingsResponse { + return s.currentUser +} + +func (s *SessionLoader) isAuthenticated() bool { + return s.currentUser != nil && s.currentUser.ScreenName != "" +} + +func (s *SessionLoader) LoadPage(url string) error { + mainPageUrl, err := neturl.Parse(url) + if err != nil { + return err + } + extraHeaders := map[string]string{ + "upgrade-insecure-requests": "1", + "sec-fetch-site": "none", + "sec-fetch-user": "?1", + "sec-fetch-dest": "document", + } + mainPageResp, mainPageRespBody, err := s.client.MakeRequest(url, "GET", s.client.buildHeaders(HeaderOpts{Extra: extraHeaders, WithCookies: true}), nil, types.NONE) + if err != nil { + return err + } + + s.client.cookies.UpdateFromResponse(mainPageResp) + if s.client.cookies.IsCookieEmpty(cookies.XGuestID) { + s.client.Logger.Err(errCookieGuestIDNotFound).Msg("No GuestID found in response headers") + return errCookieGuestIDNotFound + } + + mainPageHTML := string(mainPageRespBody) + migrationUrl, migrationRequired := methods.ParseMigrateURL(mainPageHTML) + if migrationRequired { + extraHeaders = map[string]string{ + "upgrade-insecure-requests": "1", + "sec-fetch-site": "cross-site", + "sec-fetch-mode": "navigate", + "sec-fetch-dest": "document", + } + migrationPageResp, migrationPageRespBody, err := s.client.MakeRequest(migrationUrl, "GET", s.client.buildHeaders(HeaderOpts{Extra: extraHeaders, Referer: fmt.Sprintf("https://%s/", mainPageUrl.Host), WithCookies: true}), nil, types.NONE) + if err != nil { + return err + } + + migrationPageHTML := string(migrationPageRespBody) + migrationFormUrl, migrationFormPayload := methods.ParseMigrateRequestData(migrationPageHTML) + if migrationFormPayload != nil { + migrationForm, err := query.Values(migrationFormPayload) + if err != nil { + return err + } + migrationPayload := []byte(migrationForm.Encode()) + extraHeaders["origin"] = endpoints.TWITTER_BASE_URL + + s.client.disableRedirects() + mainPageResp, _, err = s.client.MakeRequest(migrationFormUrl, "POST", s.client.buildHeaders(HeaderOpts{Extra: extraHeaders, Referer: endpoints.TWITTER_BASE_URL + "/", WithCookies: true}), migrationPayload, types.FORM) + if err == nil && !errors.Is(err, ErrRedirectAttempted) { + return fmt.Errorf("failed to make request to main page, server did not respond with a redirect response") + } + s.client.enableRedirects() + s.client.cookies.UpdateFromResponse(mainPageResp) // update the cookies received from the redirected response headers + + migrationFormUrl = endpoints.BASE_URL + mainPageResp.Header.Get("Location") + mainPageResp, mainPageRespBody, err = s.client.MakeRequest(migrationFormUrl, "GET", s.client.buildHeaders(HeaderOpts{Extra: extraHeaders, Referer: endpoints.TWITTER_BASE_URL + "/", WithCookies: true}), migrationPayload, types.FORM) + if err != nil { + return err + } + + mainPageHTML := string(mainPageRespBody) + err = s.client.parseMainPageHTML(mainPageResp, mainPageHTML) + if err != nil { + return err + } + + err = s.doInitialClientLoggingEvents() + if err != nil { + return err + } + + } else { + return fmt.Errorf("failed to find form request data in migration response: (response_body=%s, status_code=%d)", migrationPageHTML, migrationPageResp.StatusCode) + } + } else { + // most likely means... already authenticated + mainPageHTML := string(mainPageRespBody) + err = s.client.parseMainPageHTML(mainPageResp, mainPageHTML) + if err != nil { + return err + } + + err = s.doInitialClientLoggingEvents() + if err != nil { + return err + } + } + return nil +} + +func (s *SessionLoader) doCookiesMetaDataLoad() error { + logData := []interface{}{ + &payload.JotLogPayload{Description: "rweb:cookiesMetadata:load", Product: "rweb", EventValue: methods.GenerateEventValue()}, + } + return s.client.performJotClientEvent(payload.JotLoggingCategoryPerftown, false, logData) +} + +func (s *SessionLoader) doInitialClientLoggingEvents() error { + err := s.doCookiesMetaDataLoad() + if err != nil { + return err + } + country := s.GetCountry() + logData := []interface{}{ + &payload.JotLogPayload{ + Description: "rweb:init:storePrepare", + Product: "rweb", + DurationMS: 9, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:perfSupported", + Product: "rweb", + DurationMS: 1, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:perfSupported:" + country, + Product: "rweb", + DurationMS: 1, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:connect", + Product: "rweb", + DurationMS: 165, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:connect:" + country, + Product: "rweb", + DurationMS: 165, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:process", + Product: "rweb", + DurationMS: 177, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:process:" + country, + Product: "rweb", + DurationMS: 177, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:response", + Product: "rweb", + DurationMS: 212, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:response:" + country, + Product: "rweb", + DurationMS: 212, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:interactivity", + Product: "rweb", + DurationMS: 422, + }, + &payload.JotLogPayload{ + Description: "rweb:ttft:interactivity:" + country, + Product: "rweb", + DurationMS: 422, + }, + } + err = s.client.performJotClientEvent(payload.JotLoggingCategoryPerftown, false, logData) + if err != nil { + return err + } + + triggeredTimestamp := methods.GetTimestampMS() + logData = []interface{}{ + &payload.JotDebugLogPayload{ + Category: payload.JotDebugLoggingCategoryClientEvent, + TriggeredOn: triggeredTimestamp, + FormatVersion: 2, + Items: []interface{}{}, + EventNamespace: payload.EventNamespace{ + Page: "cookie_compliance_banner", + Action: "impression", + Client: "m5", + }, + ClientEventSequenceStartTimestamp: triggeredTimestamp, + ClientEventSequenceNumber: 0, + ClientAppID: "3033300", + }, + } + + err = s.client.performJotClientEvent("", true, logData) + if err != nil { + return err + } + + return nil +} + +func (s *SessionLoader) GetCountry() string { + return s.country +} + +func (s *SessionLoader) SetCountry(country string) { + s.country = country +} + +func (s *SessionLoader) SetVerificationToken(verificationToken string) { + s.verificationToken = verificationToken +} + +func (s *SessionLoader) GetVerificationToken() string { + return s.verificationToken +} + +func (s *SessionLoader) SetAuthTokens(authenticatedToken, notAuthenticatedToken string) { + s.authTokens.authenticated = authenticatedToken + s.authTokens.notAuthenticated = notAuthenticatedToken +} + +func (s *SessionLoader) GetAuthTokens() *SessionAuthTokens { + return s.authTokens +} diff --git a/pkg/twittermeow/types/http.go b/pkg/twittermeow/types/http.go new file mode 100644 index 0000000..af130c6 --- /dev/null +++ b/pkg/twittermeow/types/http.go @@ -0,0 +1,9 @@ +package types + +type ContentType string + +const ( + NONE ContentType = "" + JSON ContentType = "application/json" + FORM ContentType = "application/x-www-form-urlencoded" +) diff --git a/pkg/twittermeow/types/onboarding.go b/pkg/twittermeow/types/onboarding.go new file mode 100644 index 0000000..f16e237 --- /dev/null +++ b/pkg/twittermeow/types/onboarding.go @@ -0,0 +1,532 @@ +package types + +type TaskResponse struct { + FlowToken string `json:"flow_token,omitempty"` + Status string `json:"status,omitempty"` + Subtasks []Subtasks `json:"subtasks,omitempty"` +} +type ScribeConfig struct { + Action string `json:"action,omitempty"` + Component string `json:"component,omitempty"` + Page string `json:"page,omitempty"` + Section string `json:"section,omitempty"` +} +type Callbacks struct { + Endpoint string `json:"endpoint,omitempty"` + Trigger string `json:"trigger,omitempty"` + ScribeConfig ScribeConfig `json:"scribe_config,omitempty"` +} + +type Entities struct { + FromIndex int `json:"from_index,omitempty"` + NavigationLink NavigationLink `json:"navigation_link,omitempty"` + ToIndex int `json:"to_index,omitempty"` +} +type DetailText struct { + Entities []Entities `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type PrimaryText struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type Header struct { + PrimaryText PrimaryText `json:"primary_text,omitempty"` +} +type SecondaryText struct { + Entities []Entities `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type Icon struct { + Icon string `json:"icon,omitempty"` +} +type NavigationLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` +} +type Button struct { + Icon Icon `json:"icon,omitempty"` + NavigationLink NavigationLink `json:"navigation_link,omitempty"` + PreferredSize string `json:"preferred_size,omitempty"` + Style string `json:"style,omitempty"` +} +type Message struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type InAppNotification struct { + Message Message `json:"message,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + WaitTimeMs int `json:"wait_time_ms,omitempty"` +} +type Link struct { + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` +} +type OpenLink struct { + Link Link `json:"link,omitempty"` + OnboardingCallbackPath string `json:"onboarding_callback_path,omitempty"` +} +type Text struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type ProgressIndication struct { + Text Text `json:"text,omitempty"` +} +type EmailNextLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` +} +type JsInstrumentation struct { + URL string `json:"url,omitempty"` +} +type SignUp struct { + AllowedIdentifiers string `json:"allowed_identifiers,omitempty"` + BirthdayExplanation string `json:"birthday_explanation,omitempty"` + BirthdayHint string `json:"birthday_hint,omitempty"` + BirthdayType string `json:"birthday_type,omitempty"` + EmailHint string `json:"email_hint,omitempty"` + EmailNextLink EmailNextLink `json:"email_next_link,omitempty"` + IgnoreBirthday bool `json:"ignore_birthday,omitempty"` + JsInstrumentation JsInstrumentation `json:"js_instrumentation,omitempty"` + NameHint string `json:"name_hint,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PasswordHint string `json:"password_hint,omitempty"` + PhoneEmailHint string `json:"phone_email_hint,omitempty"` + PhoneHint string `json:"phone_hint,omitempty"` + PrimaryText string `json:"primary_text,omitempty"` + UseDevicePrefill bool `json:"use_device_prefill,omitempty"` + UseEmailText string `json:"use_email_text,omitempty"` + UsePhoneText string `json:"use_phone_text,omitempty"` +} +type BooleanData struct { + InitialValue bool `json:"initial_value,omitempty"` +} +type ValueData struct { + BooleanData BooleanData `json:"boolean_data,omitempty"` +} +type Settings struct { + PrimaryText PrimaryText `json:"primary_text,omitempty"` + ValueType string `json:"value_type,omitempty"` + ValueData ValueData `json:"value_data,omitempty"` + ValueIdentifier string `json:"value_identifier,omitempty"` +} +type SettingsList0 struct { + DetailText DetailText `json:"detail_text,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` + Settings []Settings `json:"settings,omitempty"` +} +type SubtaskDataReference struct { + Key string `json:"key,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` +} +type Birthday struct { + SubtaskDataReference SubtaskDataReference `json:"subtask_data_reference,omitempty"` +} +type SubtaskNavigationContext struct { + Action string `json:"action,omitempty"` +} +type BirthdayEditLink struct { + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type BirthdayRequirement struct { + Day int `json:"day,omitempty"` + Month int `json:"month,omitempty"` + Year int `json:"year,omitempty"` +} +type Email struct { + SubtaskDataReference SubtaskDataReference `json:"subtask_data_reference,omitempty"` +} +type EmailEditLink struct { + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type InvalidBirthdayLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SuppressClientEvents bool `json:"suppress_client_events,omitempty"` +} +type Name struct { + SubtaskDataReference SubtaskDataReference `json:"subtask_data_reference,omitempty"` +} +type NameEditLink struct { + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type PhoneEditLink struct { + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type PhoneNextLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` +} +type PhoneNumber struct { + SubtaskDataReference SubtaskDataReference `json:"subtask_data_reference,omitempty"` +} +type SignUpReview struct { + Birthday Birthday `json:"birthday,omitempty"` + BirthdayEditLink BirthdayEditLink `json:"birthday_edit_link,omitempty"` + BirthdayHint string `json:"birthday_hint,omitempty"` + BirthdayRequirement BirthdayRequirement `json:"birthday_requirement,omitempty"` + BirthdayType string `json:"birthday_type,omitempty"` + DetailText DetailText `json:"detail_text,omitempty"` + Email Email `json:"email,omitempty"` + EmailEditLink EmailEditLink `json:"email_edit_link,omitempty"` + EmailHint string `json:"email_hint,omitempty"` + EmailNextLink EmailNextLink `json:"email_next_link,omitempty"` + IgnoreBirthday bool `json:"ignore_birthday,omitempty"` + InvalidBirthdayLink InvalidBirthdayLink `json:"invalid_birthday_link,omitempty"` + Name Name `json:"name,omitempty"` + NameEditLink NameEditLink `json:"name_edit_link,omitempty"` + NameHint string `json:"name_hint,omitempty"` + PhoneEditLink PhoneEditLink `json:"phone_edit_link,omitempty"` + PhoneHint string `json:"phone_hint,omitempty"` + PhoneNextLink PhoneNextLink `json:"phone_next_link,omitempty"` + PhoneNumber PhoneNumber `json:"phone_number,omitempty"` + PrimaryText string `json:"primary_text,omitempty"` +} +type DiscoverableByEmailDetailText struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type DiscoverableByPhoneDetailText struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type PrivacyOptions struct { + DiscoverableByEmail bool `json:"discoverable_by_email,omitempty"` + DiscoverableByEmailDetailText DiscoverableByEmailDetailText `json:"discoverable_by_email_detail_text,omitempty"` + DiscoverableByEmailLabel string `json:"discoverable_by_email_label,omitempty"` + DiscoverableByPhone bool `json:"discoverable_by_phone,omitempty"` + DiscoverableByPhoneDetailText DiscoverableByPhoneDetailText `json:"discoverable_by_phone_detail_text,omitempty"` + DiscoverableByPhoneLabel string `json:"discoverable_by_phone_label,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PrimaryText string `json:"primary_text,omitempty"` + SecondaryText string `json:"secondary_text,omitempty"` +} +type CancelLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type AlertDialog struct { + CancelLink CancelLink `json:"cancel_link,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` + SecondaryText SecondaryText `json:"secondary_text,omitempty"` +} +type PrimaryActionLinks struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` + IsDestructive bool `json:"is_destructive,omitempty"` +} +type MenuDialog struct { + CancelLink CancelLink `json:"cancel_link,omitempty"` + DismissLink DismissLink `json:"dismiss_link,omitempty"` + PrimaryActionLinks []PrimaryActionLinks `json:"primary_action_links,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` +} +type Subtext struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` +} +type Choices struct { + Icon Icon `json:"icon,omitempty"` + ID string `json:"id,omitempty"` + Subtext Subtext `json:"subtext,omitempty"` + Text Text `json:"text,omitempty"` +} +type NextLinkOptions struct { + MinEnableCount int `json:"min_enable_count,omitempty"` +} +type SkipLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` +} +type ChoiceSelection struct { + Choices []Choices `json:"choices,omitempty"` + DetailText DetailText `json:"detail_text,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + NextLinkOptions NextLinkOptions `json:"next_link_options,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` + SecondaryText SecondaryText `json:"secondary_text,omitempty"` + Sections []any `json:"sections,omitempty"` + SelectedChoices []string `json:"selected_choices,omitempty"` + SelectionType string `json:"selection_type,omitempty"` + SkipLink SkipLink `json:"skip_link,omitempty"` + Style string `json:"style,omitempty"` +} +type PhoneVerification struct { + AutoVerifyHintText string `json:"auto_verify_hint_text,omitempty"` + CancelLink CancelLink `json:"cancel_link,omitempty"` + DetailText DetailText `json:"detail_text,omitempty"` + HintText string `json:"hint_text,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PhoneNumber PhoneNumber `json:"phone_number,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` + SecondaryText SecondaryText `json:"secondary_text,omitempty"` + SendViaVoice bool `json:"send_via_voice,omitempty"` +} +type EmailVerification struct { + CancelLink CancelLink `json:"cancel_link,omitempty"` + DetailText DetailText `json:"detail_text,omitempty"` + Email Email `json:"email,omitempty"` + HintText string `json:"hint_text,omitempty"` + Name Name `json:"name,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` + SecondaryText SecondaryText `json:"secondary_text,omitempty"` + VerificationStatusPollingEnabled bool `json:"verification_status_polling_enabled,omitempty"` +} +type DismissLink struct { + IsDestructive bool `json:"is_destructive,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` + SuppressClientEvents bool `json:"suppress_client_events,omitempty"` +} +type NextLink struct { + IsDestructive bool `json:"is_destructive,omitempty"` + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + SubtaskNavigationContext SubtaskNavigationContext `json:"subtask_navigation_context,omitempty"` + SuppressClientEvents bool `json:"suppress_client_events,omitempty"` +} +type AlertDialogSuppressClientEvents struct { + DismissLink DismissLink `json:"dismiss_link,omitempty"` + NextLink NextLink `json:"next_link,omitempty"` + PrimaryText PrimaryText `json:"primary_text,omitempty"` +} +type Subtasks struct { + Callbacks []Callbacks `json:"callbacks,omitempty"` + SubtaskBackNavigation string `json:"subtask_back_navigation,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + InAppNotification InAppNotification `json:"in_app_notification,omitempty"` + OpenLink OpenLink `json:"open_link,omitempty"` + ProgressIndication ProgressIndication `json:"progress_indication,omitempty"` + SignUp SignUp `json:"sign_up,omitempty"` + SettingsList SettingsList0 `json:"settings_list,omitempty"` + //SettingsList1 SettingsList1 `json:"settings_list,omitempty"` + SignUpReview SignUpReview `json:"sign_up_review,omitempty"` + PrivacyOptions PrivacyOptions `json:"privacy_options,omitempty"` + AlertDialog AlertDialog `json:"alert_dialog,omitempty"` + MenuDialog MenuDialog `json:"menu_dialog,omitempty"` + ChoiceSelection ChoiceSelection `json:"choice_selection,omitempty"` + PhoneVerification PhoneVerification `json:"phone_verification,omitempty"` + EmailVerification EmailVerification `json:"email_verification,omitempty"` + AlertDialogSuppressClientEvents AlertDialogSuppressClientEvents `json:"alert_dialog_suppress_client_events,omitempty"` + SelectAvatar *SelectAvatar `json:"select_avatar,omitempty"` +} + +type CallbackPayload struct { + Product string `json:"product,omitempty"` + Identifier string `json:"identifier,omitempty"` + Params string `json:"params,omitempty"` + TimestampMs int64 `json:"timestampMs,omitempty"` +} + +type UploadMediaStartResponse struct { + MediaID int64 `json:"media_id,omitempty"` + MediaIDString string `json:"media_id_string,omitempty"` + Size int `json:"size,omitempty"` + ExpiresAfterSecs int `json:"expires_after_secs,omitempty"` + Image struct { + ImageType string `json:"image_type,omitempty"` + W int `json:"w,omitempty"` + H int `json:"h,omitempty"` + } `json:"image,omitempty"` +} + +type SelectAvatar struct { + NextLink struct { + Callbacks []struct { + Endpoint string `json:"endpoint,omitempty"` + Trigger string `json:"trigger,omitempty"` + ScribeConfig struct { + Action string `json:"action,omitempty"` + Component string `json:"component,omitempty"` + Element string `json:"element,omitempty"` + Page string `json:"page,omitempty"` + Section string `json:"section,omitempty"` + } `json:"scribe_config,omitempty"` + } `json:"callbacks,omitempty"` + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + } `json:"next_link,omitempty"` + PrimaryText struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` + } `json:"primary_text,omitempty"` + SecondaryText struct { + Entities []any `json:"entities,omitempty"` + Text string `json:"text,omitempty"` + } `json:"secondary_text,omitempty"` + SkipLink struct { + Label string `json:"label,omitempty"` + LinkID string `json:"link_id,omitempty"` + LinkType string `json:"link_type,omitempty"` + SubtaskID string `json:"subtask_id,omitempty"` + } `json:"skip_link,omitempty"` +} + +type UserRecommendationsOnboardingResponse struct { + GlobalObjects struct { + Tweets struct { + } `json:"tweets,omitempty"` + Users map[string]*RecommendedUser `json:"users,omitempty"` + Moments struct { + } `json:"moments,omitempty"` + Cards struct { + } `json:"cards,omitempty"` + Places struct { + } `json:"places,omitempty"` + Media struct { + } `json:"media,omitempty"` + Broadcasts struct { + } `json:"broadcasts,omitempty"` + Topics struct { + } `json:"topics,omitempty"` + Lists struct { + } `json:"lists,omitempty"` + } `json:"globalObjects,omitempty"` + Timeline struct { + ID string `json:"id,omitempty"` + Instructions []struct { + AddEntries struct { + Entries []struct { + EntryID string `json:"entryId,omitempty"` + SortIndex string `json:"sortIndex,omitempty"` + Content struct { + TimelineModule struct { + Items []struct { + EntryID string `json:"entryId,omitempty"` + Item struct { + Content struct { + User struct { + ID string `json:"id,omitempty"` + DisplayType string `json:"displayType,omitempty"` + EnableReactiveBlending bool `json:"enableReactiveBlending,omitempty"` + } `json:"user,omitempty"` + } `json:"content,omitempty"` + ClientEventInfo struct { + Component string `json:"component,omitempty"` + Element string `json:"element,omitempty"` + Details struct { + TimelinesDetails struct { + SourceData string `json:"sourceData,omitempty"` + } `json:"timelinesDetails,omitempty"` + } `json:"details,omitempty"` + Action string `json:"action,omitempty"` + } `json:"clientEventInfo,omitempty"` + } `json:"item,omitempty"` + } `json:"items,omitempty"` + DisplayType string `json:"displayType,omitempty"` + Header struct { + Text string `json:"text,omitempty"` + DisplayType string `json:"displayType,omitempty"` + } `json:"header,omitempty"` + } `json:"timelineModule,omitempty"` + } `json:"content,omitempty"` + } `json:"entries,omitempty"` + } `json:"addEntries,omitempty"` + } `json:"instructions,omitempty"` + } `json:"timeline,omitempty"` +} + +type RecommendedUser struct { + ID int `json:"id,omitempty"` + IDStr string `json:"id_str,omitempty"` + Name string `json:"name,omitempty"` + ScreenName string `json:"screen_name,omitempty"` + Location string `json:"location,omitempty"` + Description string `json:"description,omitempty"` + URL string `json:"url,omitempty"` + Entities struct { + URL struct { + Urls []struct { + URL string `json:"url,omitempty"` + ExpandedURL string `json:"expanded_url,omitempty"` + DisplayURL string `json:"display_url,omitempty"` + Indices []int `json:"indices,omitempty"` + } `json:"urls,omitempty"` + } `json:"url,omitempty"` + Description struct { + Urls []any `json:"urls,omitempty"` + } `json:"description,omitempty"` + } `json:"entities,omitempty"` + Protected bool `json:"protected,omitempty"` + FollowersCount int `json:"followers_count,omitempty"` + FriendsCount int `json:"friends_count,omitempty"` + ListedCount int `json:"listed_count,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + FavouritesCount int `json:"favourites_count,omitempty"` + UtcOffset any `json:"utc_offset,omitempty"` + TimeZone any `json:"time_zone,omitempty"` + GeoEnabled bool `json:"geo_enabled,omitempty"` + Verified bool `json:"verified,omitempty"` + StatusesCount int `json:"statuses_count,omitempty"` + Lang any `json:"lang,omitempty"` + ContributorsEnabled bool `json:"contributors_enabled,omitempty"` + IsTranslator bool `json:"is_translator,omitempty"` + IsTranslationEnabled bool `json:"is_translation_enabled,omitempty"` + ProfileBackgroundColor string `json:"profile_background_color,omitempty"` + ProfileBackgroundImageURL string `json:"profile_background_image_url,omitempty"` + ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https,omitempty"` + ProfileBackgroundTile bool `json:"profile_background_tile,omitempty"` + ProfileImageURL string `json:"profile_image_url,omitempty"` + ProfileImageURLHTTPS string `json:"profile_image_url_https,omitempty"` + ProfileBannerURL string `json:"profile_banner_url,omitempty"` + ProfileLinkColor string `json:"profile_link_color,omitempty"` + ProfileSidebarBorderColor string `json:"profile_sidebar_border_color,omitempty"` + ProfileSidebarFillColor string `json:"profile_sidebar_fill_color,omitempty"` + ProfileTextColor string `json:"profile_text_color,omitempty"` + ProfileUseBackgroundImage bool `json:"profile_use_background_image,omitempty"` + HasExtendedProfile bool `json:"has_extended_profile,omitempty"` + DefaultProfile bool `json:"default_profile,omitempty"` + DefaultProfileImage bool `json:"default_profile_image,omitempty"` + CanMediaTag any `json:"can_media_tag,omitempty"` + Following bool `json:"following,omitempty"` + FollowRequestSent bool `json:"follow_request_sent,omitempty"` + Notifications bool `json:"notifications,omitempty"` + Muting bool `json:"muting,omitempty"` + Blocking bool `json:"blocking,omitempty"` + BlockedBy bool `json:"blocked_by,omitempty"` + TranslatorType string `json:"translator_type,omitempty"` + WithheldInCountries []any `json:"withheld_in_countries,omitempty"` + FollowedBy bool `json:"followed_by,omitempty"` + ExtIsBlueVerified bool `json:"ext_is_blue_verified,omitempty"` +}