Add activitystreams.ActorActivity for Actor updates

Bump golang.org/x/net from 0.54.0 to 0.55.0

activitystreams/activity.go

@@ -4,14 +4,20 @@ package activitystreams
 
 import (
 	"time"
+
+	"github.com/writeas/web-core/id"
 )
 
 const (
 	Namespace = "https://www.w3.org/ns/activitystreams"
-	toPublic  = "https://www.w3.org/ns/activitystreams#Public"
+	ToPublic  = "https://www.w3.org/ns/activitystreams#Public"
 )
 
-var Extensions = map[string]string{}
+var (
+	Extensions = map[string]string{}
+
+	apMonetizationContext = "https://webmonetization.org/ns.jsonld"
+)
 
 // Activity describes actions that have either already occurred, are in the
 // process of occurring, or may occur in the future.
@@ -19,6 +25,7 @@ type Activity struct {
 	BaseObject
 	Actor     string     `json:"actor"`
 	Published time.Time  `json:"published,omitempty"`
+	Updated   *time.Time `json:"updated,omitempty"`
 	To        []string   `json:"to,omitempty"`
 	CC        []string   `json:"cc,omitempty"`
 	Object    *Object    `json:"object"`
@@ -68,6 +75,43 @@ func NewUpdateActivity(o *Object) *Activity {
 		Object:    o,
 		Published: o.Published,
 	}
+	if o.Updated != nil && !o.Updated.IsZero() {
+		a.Updated = o.Updated
+	}
+	return &a
+}
+
+// ActorActivity describes an action whose object is a full actor (Person),
+// such as an Update to a profile's name, summary, or icon. It parallels
+// Activity, which instead carries a content *Object.
+type ActorActivity struct {
+	BaseObject
+	Actor   string     `json:"actor"`
+	Updated *time.Time `json:"updated,omitempty"`
+	To      []string   `json:"to,omitempty"`
+	CC      []string   `json:"cc,omitempty"`
+	Object  *Person    `json:"object"`
+}
+
+// NewUpdateActorActivity builds an Update activity whose object is the given
+// actor (Person), used to federate profile changes such as the actor's name,
+// summary, or icon. It addresses the public namespace and CCs the actor's
+// followers collection. The activity gets a unique ID derived from the actor's
+// IRI so remote instances don't dedupe repeated updates against one another.
+func NewUpdateActorActivity(p *Person) *ActorActivity {
+	a := ActorActivity{
+		BaseObject: BaseObject{
+			Context: []interface{}{
+				Namespace,
+			},
+			ID:   p.ID + "#update-" + id.GenerateFriendlyRandomString(20),
+			Type: "Update",
+		},
+		Actor:  p.ID,
+		To:     []string{ToPublic},
+		CC:     []string{p.Followers},
+		Object: p,
+	}
 	return &a
 }
 
@@ -107,6 +151,7 @@ func NewFollowActivity(actorIRI, followeeIRI string) *FollowActivity {
 type Object struct {
 	BaseObject
 	Published    time.Time         `json:"published,omitempty"`
+	Updated      *time.Time        `json:"updated,omitempty"`
 	Summary      *string           `json:"summary,omitempty"`
 	InReplyTo    *string           `json:"inReplyTo,omitempty"`
 	URL          string            `json:"url"`
@@ -118,6 +163,7 @@ type Object struct {
 	ContentMap   map[string]string `json:"contentMap,omitempty"`
 	Tag          []Tag             `json:"tag,omitempty"`
 	Attachment   []Attachment      `json:"attachment,omitempty"`
+	Preview      *Object           `json:"preview,omitempty"`
 
 	// Person
 	Inbox             string     `json:"inbox,omitempty"`
@@ -130,7 +176,7 @@ type Object struct {
 	Endpoints         *Endpoints `json:"endpoints,omitempty"`
 
 	// Extensions
-	// NOTE: add extensions here
+	Monetization string `json:"monetization,omitempty"`
 }
 
 // NewNoteObject creates a basic Note object that includes the public
@@ -139,9 +185,12 @@ func NewNoteObject() *Object {
 	o := Object{
 		BaseObject: BaseObject{
 			Type: "Note",
+			Context: []interface{}{
+				Namespace,
+			},
 		},
 		To: []string{
-			toPublic,
+			ToPublic,
 		},
 	}
 	return &o
@@ -153,9 +202,12 @@ func NewArticleObject() *Object {
 	o := Object{
 		BaseObject: BaseObject{
 			Type: "Article",
+			Context: []interface{}{
+				Namespace,
+			},
 		},
 		To: []string{
-			toPublic,
+			ToPublic,
 		},
 	}
 	return &o
@@ -166,7 +218,15 @@ func NewPersonObject() *Object {
 	o := Object{
 		BaseObject: BaseObject{
 			Type: "Person",
+			Context: []interface{}{
+				Namespace,
+			},
 		},
 	}
 	return &o
 }
+
+func (o *Object) AddWebMonetization(wallet string) {
+	o.Context = append(o.Context, apMonetizationContext)
+	o.Monetization = wallet
+}

activitystreams/data.go

@@ -30,7 +30,7 @@ type (
 type OrderedCollection struct {
 	BaseObject
 	TotalItems int    `json:"totalItems"`
-	First      string `json:"first"`
+	First      string `json:"first,omitempty"`
 	Last       string `json:"last,omitempty"`
 }
 
@@ -43,9 +43,13 @@ func NewOrderedCollection(accountRoot, collType string, items int) *OrderedColle
 			ID:   accountRoot + "/" + collType,
 			Type: "OrderedCollection",
 		},
-		First:      accountRoot + "/" + collType + "?page=1",
 		TotalItems: items,
 	}
+	if items > 0 {
+		// Next page should only be indicated if there are potentially items on it. NOTE: with no knowledge of items-
+		// per-page, this still may need to be altered by the func caller.
+		oc.First = accountRoot + "/" + collType + "?page=1"
+	}
 	return &oc
 }
 

activitystreams/person.go

@@ -13,6 +13,9 @@ type Person struct {
 	Summary           string    `json:"summary"`
 	PublicKey         PublicKey `json:"publicKey"`
 	Endpoints         Endpoints `json:"endpoints"`
+
+	// Extensions
+	Monetization string `json:"monetization,omitempty"`
 }
 
 func NewPerson(accountRoot string) *Person {
@@ -49,3 +52,8 @@ func (p *Person) SetPrivKey(k []byte) {
 func (p *Person) GetPrivKey() []byte {
 	return p.PublicKey.privateKey
 }
+
+func (p *Person) AddWebMonetization(wallet string) {
+	p.Context = append(p.Context, apMonetizationContext)
+	p.Monetization = wallet
+}

bots/bots.go

@@ -7,140 +7,272 @@ import "strings"
 
 var bots = map[string]bool{
 	"ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)":                                 true,
+	"AcademicBotRTU (https://academicbot.rtu.lv; mailto:caps@rtu.lv)":          true,
 	"AddThis.com robot tech.support@clearspring.com":                           true,
+	"AdsBot-Google (+http://www.google.com/adsbot.html)":                       true,
+	"AdsTxtCrawlerTP/1.2":                                                      true,
 	"Alwyzbot/1.0":                                                             true,
-	"Apache-HttpClient/4.2.2 (java 1.5)":                                       true,
+	"Amazon-Advertising-ad-standards-bot/1.0":                                  true,
+	"Apple Color Emoji":                                                        true,
+	"AwarioRssBot/1.0 (+https://awario.com/bots.html; bots@awario.com)":        true,
+	"AwarioSmartBot/1.0 (+https://awario.com/bots.html; bots@awario.com)":      true,
+	"BeeperBot/0 Matrix-Media-Repo/1":                                          true,
+	"bidswitchbot/1.0":                                                         true,
 	"bitlybot":                                                                 true,
-	"bitlybot/3.0 (+http://bit.ly/)":                                           true,
+	"BrightEdge Crawler/1.0 (crawler@brightedge.com)":                          true,
+	"BSbot 1.1 (monthly copyright check - html/js/css)":                        true,
 	"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true,
+	"CheckMarkNetwork/1.0 (+http://www.checkmarknetwork.com/spider.html)":      true,
+	"Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
 	"cis455crawler":                                         true,
+	"Clickagy Intelligence Bot v2":                          true,
 	"COMODO SSL Checker":                                    true,
+	"companyspotter/1.0.0.0 (robot@companyspotter.com)":     true,
+	"crawler_eb_germany_2.0":                                true,
 	"crawlernutchtest/Nutch-1.9":                            true,
 	"CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true,
 	"CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true,
 	"datebot":                                     true,
 	"DeadYetBot/1.0 (+http://deadyet.lol)":        true,
+	"Deskyobot/1.0 (+https://www.deskyo.com/bot)": true,
+	"DingTalkBot-LinkService/1.0 (+https://open-doc.dingtalk.com/microapp/faquestions/ftpfeu)":            true,
+	"DisqusAdstxtCrawler/0.1 (+https://help.disqus.com/en/articles/1765357-ads-txt-implementation-guide)": true,
 	"dj-research/Nutch-1.11 (analytics@@demandjump.com)":                                                  true,
+	"DnBCrawler-Analytics": true,
 	"DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
-	"DomainCrawler/1.0": true,
 	"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com":                  true,
+	"DomainCrawler/1.0": true,
+	"DomainStatsBot/1.0 (https://domainstats.com/pages/our-bot)":                                  true,
+	"drinespider/Nutch-1.19 (D-RINE Spider; www.d-rine.com/search/about; www.d-rine.com/contact)": true,
+	"DuckDuckBot-Https/1.1; (+https://duckduckgo.com/duckduckbot)":                                true,
 	"ExactSeekCrawler/1.0": true,
+	"EyeMonIT Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36": true,
+	"FediCrawl/1.0":                     true,
+	"fedistatsCrawler/1.0":              true,
+	"FeedlyBot/1.0 (http://feedly.com)": true,
+	"GG PeekBot 2.0 ( https://www.gg.pl/ https://www.gg.pl/info/praca/ )": true,
 	"Gigabot/1.0": true,
+	"GNUsocialBot 2.0.1-beta0 - https://gnusocial.rocks":          true,
 	"Go 1.1 package http":                                         true,
-	"Googlebot/2.1 (+http://www.googlebot.com/bot.html)":                                  true,
+	"Go-http-client/2.0":                                          true,
-	"Googlebot/2.1 (+http://www.google.com/bot.html)":                                     true,
+	"Google-Adwords-Instant (+http://www.google.com/adsbot.html)": true,
-	"Googlebot-Image/1.0":                                                                 true,
+	"Google-Display-Ads-Bot":                                      true,
-	"libwww-perl/6.04":                                                                    true,
+	"Googlebot":                                                   true,
+	"Greppr Bot 1.0/Nutch-1.19":                                   true,
+	"GroupMeBot/1.0":                                              true,
+	"GuzzleHttp/7":                                                true,
+	"Gwene/1.0 (The gwene.org rss-to-news gateway) Googlebot":     true,
+	"HubSpot Connect 2.0 (http://dev.hubspot.com/) (namespace: hs_web_crawler) - RoadsModelsJobs-hubspot-53-domain-web-crawl":                         true,
+	"ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)":                                                                     true,
+	"IABot/2.0 (+https://meta.wikimedia.org/wiki/InternetArchiveBot/FAQ_for_sysadmins) (Checking if link from Wikipedia is broken and needs removal)": true,
+	"IonCrawl (https://www.ionos.de/terms-gtc/faq-crawler-en/)":                                                                                       true,
+	"LightspeedSystemsCrawler Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)":                                                              true,
+	"LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)":                                                           true,
 	"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)":                                              true,
 	"LivelapBot/0.2 (http://site.livelap.com/crawler)":                                                                                                true,
 	"LSSRocketCrawler/1.0 LightspeedSystems":                                                                                                          true,
+	"magpie-crawler/1.1 (U; Linux amd64; en-GB; +http://www.brandwatch.net)":                                                                          true,
+	"Mastodon server indexer":                            true,
+	"MBCrawler/1.0 (https://monitorbacklinks.com/robot)": true,
 	"Mediatoolkitbot (complaints@mediatoolkit.com)":      true,
 	"Mediatoolkitbot (info@mediatoolkit.com)":            true,
-	"Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)":                                                true,
 	"Melvil/1.0": true,
 	"MetaCommentBot; http://metacomment.io/about":                            true,
 	"mfibot/1.1 (http://www.mfisoft.ru/analyst/; <admin@mfisoft.ru>; en-RU)": true,
-	"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                                                true,
+	"Minoru's Fediverse Crawler (+https://nodes.fediverse.party)":            true,
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)":                          true,
+	"mj12bot": true,
-	"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)":                                                                                                                                                          true,
+	"Mozilla/4.0 (compatible; QwertyNetworksBot/1.0; +https://qwertynetworks.com/faq/QwertyNetworksBot)":                                                              true,
+	"Mozilla/4.0 (compatible; Wibybot; https://wiby.me/)":                                                                                                             true,
+	"Mozilla/5.0 (Android 11; Mobile; rv:109.0) Gecko/114.0 Firefox/114.0":                                                                                            true,
+	"Mozilla/5.0 (compatible) SemanticScholarBot (+https://www.semanticscholar.org/crawler)":                                                                          true,
+	"Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)":                                                                                              true,
 	"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)":                                                                                        true,
 	"Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )":                                                                         true,
+	"Mozilla/5.0 (compatible; AmazonAdBot/1.0; +https://adbot.amazon.com)":                                                                                            true,
+	"Mozilla/5.0 (compatible; archive.org_bot +http://archive.org/details/archive.org_bot) Zeno/0030699 warc/v0.8.32":                                                 true,
+	"Mozilla/5.0 (compatible; Atomseobot/2.0; +http://https://error404.atomseo.com/)":                                                                                 true,
+	"Mozilla/5.0 (compatible; AwarioBot/1.0; +https://awario.com/bots.html)":                                                                                          true,
+	"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html":                                                                              true,
 	"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)":                                                                             true,
-	"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)":                                                                                                                                                     true,
+	"Mozilla/5.0 (compatible; BitSightBot/1.0)":                                                                                                                       true,
 	"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)":                                                                                             true,
+	"Mozilla/5.0 (compatible; BomboraBot/1.0; +http://www.bombora.com/bot)":                                                                                           true,
 	"Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)":                                                                                           true,
+	"Mozilla/5.0 (compatible; Bytespider; spider-feedback@bytedance.com) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.0.0 Safari/537.36":                        true,
 	"Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)":                                                                                                  true,
+	"Mozilla/5.0 (compatible; coccocbot-image/1.0; +http://help.coccoc.com/searchengine)":                                                                             true,
+	"Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)":                                                                               true,
+	"Mozilla/5.0 (compatible; Cocolyzebot/1.0; https://cocolyze.com/bot)":                                                                                             true,
+	"Mozilla/5.0 (compatible; DataForSeoBot/1.0; +https://dataforseo.com/dataforseo-bot)":                                                                             true,
 	"Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)":                                                                                               true,
+	"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)":                                                                                               true,
+	"Mozilla/5.0 (compatible; Dmbot/1.1)":                                                                                                                             true,
 	"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)":                                                                      true,
+	"Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)":                                                                        true,
 	"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)":                                                                                   true,
+	"Mozilla/5.0 (compatible; ev-crawler/1.0; +https://headline.com/legal/crawler)":                                                                                   true,
 	"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)":                                                                                           true,
 	"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)":                                                                                      true,
+	"Mozilla/5.0 (compatible; FFZBot/2.4.1; +https://www.frankerfacez.com)":                                                                                           true,
+	"Mozilla/5.0 (compatible; FFZBot/3.0.0; +https://www.frankerfacez.com)":                                                                                           true,
 	"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)":                                                                                                true,
 	"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)":                                                                              true,
-	"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                                                                                                                                                    true,
+	"Mozilla/5.0 (compatible; GrapeshotCrawler/2.0; +http://www.grapeshot.co.uk/crawler.php)":                                                                         true,
-	"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)":                                                                                                                 true,
 	"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)":                                                           true,
-	"Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)":                                                                                                                    true,
+	"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)":                                                     true,
+	"Mozilla/5.0 (compatible; Hkfl-Bot/%s +https://hackerfall.com/)":                                                                                                  true,
+	"Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Exabot-Thumbnails)":                                                                     true,
 	"Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)":                                                                              true,
+	"Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)":                                                                                               true,
+	"Mozilla/5.0 (compatible; linkdexbot/2.0; +http://www.linkdex.com/bots/)":                                                                                         true,
 	"Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)":                                                                                         true,
 	"Mozilla/5.0 (compatible; LinkFeatureBot)":                                                                                                                        true,
-	"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)":                                                                                                                                                                     true,
 	"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321":                                               true,
+	"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)":                                                                                                         true,
 	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)":                                                                         true,
+	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Fast/2.0; +https://help.mail.ru/webmaster/indexing/robots)":                                                   true,
+	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Img/2.0; +https://help.mail.ru/webmaster/indexing/robots)":                                                    true,
 	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)":                                                                  true,
+	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +https://help.mail.ru/webmaster/indexing/robots)":                                                 true,
+	"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Target/2.0; +https://help.mail.ru/webmaster/indexing/robots)":                                                 true,
 	"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)":                                                                    true,
 	"Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)":                                                                            true,
 	"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)":                                                                                       true,
 	"Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)":                                                                                                       true,
+	"Mozilla/5.0 (compatible; MJ12bot/v1.4.2; http://www.majestic12.co.uk/bot.php?+)":                                                                                 true,
 	"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)":                                                                                 true,
+	"Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)":                                                                                                   true,
+	"Mozilla/5.0 (compatible; MojeekBot/0.11; +https://www.mojeek.com/bot.html)":                                                                                      true,
 	"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)":                                                                                       true,
 	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)":                                                                                         true,
 	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)":                                                                 true,
 	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider":                                                                                      true,
 	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)":                  true,
 	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)":                                                                                          true,
-	"Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1":                                                                                                                                                                  true,
 	"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1":                                                                               true,
 	"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0":                                                                             true,
+	"Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1":                                                                                                      true,
+	"Mozilla/5.0 (compatible; NetpeakCheckerBot/3.6; +https://netpeaksoftware.com/checker)":                                                                           true,
 	"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)":                                                                                 true,
 	"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)":                                                                                         true,
 	"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)":                                                                                    true,
 	"Mozilla/5.0 (compatible; OpsBot/1.0)":                                                                                                                            true,
 	"Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)":                                                                           true,
 	"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)":                                                             true,
+	"Mozilla/5.0 (compatible; PaperLiBot/2.1; https://support.paper.li/hc/en-us/articles/360006695637-PaperLiBot)":                                                    true,
+	"Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)":                                                                                  true,
 	"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)":                                                                                     true,
+	"Mozilla/5.0 (compatible; proximic; +https://www.comscore.com/Web-Crawler)":                                                                                       true,
+	"Mozilla/5.0 (compatible; Qwantify-dev/1.0; +https://help.qwant.com/bot/)":                                                                                        true,
+	"Mozilla/5.0 (compatible; Qwantify-dev10230/1.0; +https://help.qwant.com/bot/)":                                                                                   true,
+	"Mozilla/5.0 (compatible; Qwantify-dev19924/1.0; +https://help.qwant.com/bot/)":                                                                                   true,
+	"Mozilla/5.0 (compatible; Qwantify-dev19967/1.0; +https://help.qwant.com/bot/)":                                                                                   true,
+	"Mozilla/5.0 (compatible; Qwantify-dev2270/1.0; +https://help.qwant.com/bot/)":                                                                                    true,
+	"Mozilla/5.0 (compatible; Qwantify-dev22709/1.0; +https://help.qwant.com/bot/)":                                                                                   true,
+	"Mozilla/5.0 (compatible; Qwantify-dev31653/1.0; +https://help.qwant.com/bot/)":                                                                                   true,
+	"Mozilla/5.0 (compatible; Qwantify-dev4541/1.0; +https://help.qwant.com/bot/)":                                                                                    true,
+	"Mozilla/5.0 (compatible; Qwantify-dev5115/1.0; +https://help.qwant.com/bot/)":                                                                                    true,
+	"Mozilla/5.0 (compatible; Qwantify-dev846/1.0; +https://help.qwant.com/bot/)":                                                                                     true,
+	"Mozilla/5.0 (compatible; Qwantify-worker2994/1.0; +https://help.qwant.com/bot/)":                                                                                 true,
 	"Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*":                                                                                              true,
 	"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)":                                                                                        true,
+	"Mozilla/5.0 (compatible; rss2tg bot; +http://komar.in/en/rss2tg_crawler)":                                                                                        true,
 	"Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)":                                                                                                   true,
+	"Mozilla/5.0 (compatible; SearchMySiteBot/1.0; +https://searchmysite.net)":                                                                                        true,
+	"Mozilla/5.0 (compatible; Semanticbot/1.0; +http://sempi.tech/bot.html)":                                                                                          true,
+	"Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)":                                                                                       true,
+	"Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)":                                                                                  true,
 	"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)":                                                                                   true,
 	"Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)":                                                                                     true,
+	"Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)":                                                                                     true,
+	"Mozilla/5.0 (compatible; SemrushBot; +http://www.semrush.com/bot.html)":                                                                                          true,
 	"Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)":                                                                                    true,
 	"Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)":                                                                                  true,
 	"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)":                                                                                             true,
 	"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)":                                                                         true,
+	"Mozilla/5.0 (compatible; SeznamBot/4.0; +http://napoveda.seznam.cz/seznambot-intro/)":                                                                            true,
+	"Mozilla/5.0 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)":                                                                                   true,
 	"Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )":                                                                                        true,
 	"Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )":                                                                                        true,
 	"Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )":                                                                                        true,
 	"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )":                                                                                          true,
 	"Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)":                                                                true,
+	"Mozilla/5.0 (compatible; startmebot/1.0; +https://start.me/bot)":                                                                                                 true,
+	"Mozilla/5.0 (compatible; SurdotlyBot/1.0; +http://sur.ly/bot.html)":                                                                                              true,
+	"Mozilla/5.0 (compatible; SynapseMediaProxyBot/0.0.1; https://git.pixie.town/f0x/synapse-media-proxy)":                                                            true,
 	"Mozilla/5.0 (compatible; TestCrawler)":                                                                                                                           true,
+	"Mozilla/5.0 (compatible; TrendsmapResolver/0.1)":                                                                                                                 true,
 	"Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)":                                                                           true,
 	"Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)":                                                                                          true,
-	"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)":                                                                                                                                                      true,
+	"Mozilla/5.0 (compatible; WebwikiBot/2.1; +https://www.webwiki.de)":                                                                                               true,
+	"Mozilla/5.0 (compatible; WellKnownBot/0.1; +https://well-known.dev/about/#bot)":                                                                                  true,
 	"Mozilla/5.0 (compatible; woriobot +http://worio.com)":                                                                                                            true,
+	"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) sieve.k8s.crawler-production/1688317128-0":                                   true,
 	"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)":                                                                             true,
+	"Mozilla/5.0 (compatible; YaK/1.0; http://linkfluence.com/; bot@linkfluence.com)":                                                                                 true,
+	"Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0":                           true,
+	"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0":                                        true,
 	"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)":                                                                                                true,
+	"Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)":                                                                                           true,
+	"Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)":                                                                                             true,
+	"Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0":                         true,
+	"Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots)":                                                                                       true,
 	"Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)":                                                                                              true,
+	"Mozilla/5.0 (compatible; YetiBOT/0.1b; +http://yetibot.ovh)":                                                                                                     true,
 	"Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)":                                                                                  true,
-	"Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)":                                                                                                                                                                    true,
+	"Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)":                                                                              true,
+	"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) CriOS/56.0.2924.75 Mobile/14E5239e YisouSpider/5.0 Safari/602.1": true,
+	"Mozilla/5.0 (iPhone; CPU iPhone OS 15_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone14,3__weibo__13.6.3__iphone__os15.1)": true,
+	"Mozilla/5.0 (iPhone; CPU iPhone OS 15_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 Weibo (iPhone12,8__weibo__13.6.3__iphone__os15.6)": true,
 	"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)":                true,
-	"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                        true,
+	"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; SiteAuditBot/0.97; +http://www.semrush.com/bot.html)":            true,
-	"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0;  http://www.bing.com/bingbot.htm)":                         true,
-	"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)":                         true,
 	"Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)":                             true,
-	"Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                          true,
 	"Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4 (compatible; Laserlikebot/0.1)":                                                 true,
+	"Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)": true,
 	"Mozilla/5.0 (iPhone; CPU iPhone OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1":                                                                               true,
-	"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                         true,
+	"Mozilla/5.0 (Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0": true,
-	"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)":                      true,
+	"Mozilla/5.0 (Linux x86_64; rv:114.0) Gecko/20100101 Firefox/114.0": true,
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)":                                                      true,
+	"Mozilla/5.0 (Linux; Android 10; BAH3-W59 Build/HUAWEIBAH3-W59; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Safari/537.36 Weibo (HUAWEI-BAH3-W59__weibo__12.5.2__android__android10)":        true,
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36":                                                                                                   true,
+	"Mozilla/5.0 (Linux; Android 10; VCE-AL00 Build/HUAWEIVCE-AL00; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/92.0.4515.105 Mobile Safari/537.36 Weibo (HUAWEI-VCE-AL00__weibo__13.6.3__android__android10)": true,
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36":                                                                                                   true,
+	"Mozilla/5.0 (Linux; Android 13; V2244A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36V2244A_13_WeiboIntlAndroid_6200":                              true,
+	"Mozilla/5.0 (Linux; Android 13; V2302A Build/TP1A.220624.014; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/101.0.4951.74 Mobile Safari/537.36 V2302A_13_WeiboIntlAndroid_2612":                             true,
+	"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; https://zhanzhang.toutiao.com/)":                                                                      true,
+	"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)":                                                                       true,
+	"Mozilla/5.0 (Linux; Android 5.1.1; A37f) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.74 Mobile Safari/537.36":                                                                                                   true,
+	"Mozilla/5.0 (Linux; Android 6.0.1; vivo 1606 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 VivoBrowser/5.7.0.6":                                                         true,
+	"Mozilla/5.0 (Linux; Android 6.0.1; vivo Y66 Build/MMB29M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/87.0.4280.141 Mobile Safari/537.36 VivoBrowser/10.9.14.0":                                           true,
+	"Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)":                                                      true,
+	"Mozilla/5.0 (Linux;u;Android 4.2.2;zh-cn;) AppleWebKit/534.46 (KHTML,like Gecko) Version/5.1 Mobile Safari/10600.6.3 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)":                               true,
 	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:42.0) Gecko/20100101 Firefox/42.0":                                                                                                                                         true,
-	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36":                                                                                                    true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0 (FlipboardProxy/1.2; +http://flipboard.com/browserproxy)":                                                                                true,
 	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:28.0) Gecko/20100101 Firefox/28.0 (FlipboardProxy/1.1; +http://flipboard.com/browserproxy)":                                                                                 true,
 	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:29.0) Gecko/20100101 Firefox/29.0":                                                                                                                                          true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Applebot/0.1; +http://www.apple.com/go/applebot)":                                                     true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/601.2.4 (KHTML, like Gecko) Version/9.0.1 Safari/601.2.4 facebookexternalhit/1.1 Facebot Twitterbot/1.0":                                                        true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36":                                                                                                  true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36":                                                                                                  true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.7 DuckDuckGo/7 Safari/605.1.15":                                                                                           true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)":                                                  true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.81 Safari/537.36 Edg/104.0.1293.54; Bot":                                                                           true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36 webtru_crawler":                                                                                   true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)":                         true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.114 Safari/537.36":                                                                                                   true,
+	"Mozilla/5.0 (Macintosh; Intel Mac OS X 13_0_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 ISSCyberRiskCrawler/1.1.3":                                                                             true,
 	"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5":                                                                                            true,
+	"Mozilla/5.0 (Morningscore Bot/1.0)": true,
 	"Mozilla/5.0 (TweetmemeBot/4.0; +http://datasift.com/bot.html) Gecko/20100101 Firefox/31.0":                                                                                                                                   true,
-	"Mozilla/5.0 (Windows; Crawler; U; Windows NT 6.0; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7 (.NET CLR 3.5.30729)":                                                                                                    true,
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.5735.179 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)":                                           true,
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 (compatible; Google-Safety; +http://www.google.com/bot.html)":                                            true,
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20170101 Firefox/67.0":                   true,
 	"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36":                                                                                                              true,
 	"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0":                                                                                                                                                   true,
 	"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko":                                                                                                                                                       true,
 	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.27+ (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27":                                                                                                                         true,
 	"Mozilla/5.0 (Windows NT 5.1; rv:33.0) Gecko/20100101 Firefox/33.0":                                                                                                                                                           true,
 	"Mozilla/5.0 (Windows NT 5.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2":                                                                                                                                                         true,
-	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)":                                          true,
 	"Mozilla/5.0 (Windows NT 6.1) (compatible; SMTBot/1.0; +http://www.similartech.com/smtbot)":                                                                                                                                   true,
+	"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.89 Safari/537.1; 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)":                                          true,
+	"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 YisouSpider/5.0 Safari/537.36":                                                                                           true,
 	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36":                                                                                                               true,
 	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36":                                                                                                               true,
 	"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36":                                                                                                                true,
@@ -151,10 +283,11 @@ var bots = map[string]bool{
 	"Mozilla/5.0 (Windows NT 6.2; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0":                                                                                                                                                    true,
 	"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.131 Safari/537.36":                                                                                                               true,
 	"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0":                                                                                                                                                    true,
+	"Mozilla/5.0 (Windows; Crawler; U; Windows NT 6.0; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7 (.NET CLR 3.5.30729)":                                                                                                    true,
 	"Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.6) Gecko/20040206 Firefox/0.8":                                                                                                                                          true,
-	"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)":                                                                                   true,
 	"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36 HubSpot Webcrawler":                                                                                true,
 	"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b4) Gecko/20050908 Firefox/1.4":                                                                                                                                        true,
+	"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.13) Gecko/2009073022 Firefox/3.5.2 (.NET CLR 3.5.30729) SurveyBot/2.3 (DomainTools)":                                                                                   true,
 	"Mozilla/5.0 (Windows; U; Windows NT 5.1; ja; rv:1.8.0.10) Gecko/20070216 Firefox/1.5.0.10":                                                                                                                                   true,
 	"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.0; trendictionbot0.5.0; trendiction search; http://www.trendiction.de/bot; please let us know of any problems; web at trendiction.com) Gecko/20071127 Firefox/3.0.0.11": true,
 	"Mozilla/5.0 (Windows; U; WinNT4.0; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1":                                                                                                                                            true,
@@ -163,63 +296,133 @@ var bots = map[string]bool{
 	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.155 Safari/537.36/Gringe":                                                                                                            true,
 	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36":                                                                                                                   true,
 	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36":                                                                                                                   true,
-	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20061201 Firefox/2.0.0.8":                                                                                                                                          true,
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/61.0.3163.100 Chrome/61.0.3163.100 Safari/537.36 PingdomPageSpeed/1.0 (pingbot/2.0; +http://www.pingdom.com/)":                        true,
 	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20060118 Firefox/1.5":                                                                                                                                                  true,
+	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.8) Gecko/20061201 Firefox/2.0.0.8":                                                                                                                                          true,
 	"Mozilla/5.0 (X11; U; Linux i686; fr-FR; rv:1.8.1.6) Gecko/20080208 Ubuntu/7.10 (gutsy) Firefox/2.0.0.12":                                                                                                                     true,
 	"Mozilla/5.0 (X11; U; Linux i686; fr; rv:1.9.0.9) Gecko/2009042113 Ubuntu/8.04 (hardy) Firefox/3.0.9":                                                                                                                         true,
 	"Mozilla/5.0 (X11; U; Linux i686; hu-HU; rv:1.9.1.9) Gecko/20100330 Fedora/3.5.9-1.fc12 Firefox/3.5.9":                                                                                                                        true,
 	"Mozilla/5.0 (X11; U; Linux i686; nl; rv:1.9) Gecko/2008061015 Firefox/3.0":                                                                                                                                                   true,
 	"Mozilla/5.0 (X11; U; Linux i686; pt-BR; rv:1.8.0.6) Gecko/20060728 Firefox/1.5.0.6":                                                                                                                                          true,
 	"Mozilla/5.0 (X11; U; Linux x86_64; cs-CZ; rv:1.9.1.7) Gecko/20100106 Ubuntu/9.10 (karmic) Firefox/3.5.7":                                                                                                                     true,
-	"Mozilla/5.0 (X11; U; Linux x86_64; de; rv:1.9.2.8) Googlebot-Compatible Gecko/20100723 Ubuntu/10.04 (lucid) Firefox/3.6.8":                                                                                                   true,
 	"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.18) Gecko/20081112 Fedora/2.0.0.18-1.fc8 Firefox/2.0.0.18":                                                                                                                true,
 	"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3":                                                                                                                    true,
 	"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1":                                                                                                                                            true,
 	"Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13":                                                                                                                 true,
+	"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Xing Bot":                                                                                                                                                                   true,
+	"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot":                                                                                                                   true,
+	"Mozilla/5.0(compatible;Googlebot/2.1; +http://www.google.com/bot.html)":                                                                                                                                                      true,
+	"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)":                                                                                                                                                      true,
+	"Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)":                                                                                                                    true,
+	"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)":                                                                                                                                                          true,
+	"Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)":                                                                                                                                                                    true,
+	"Mozilla/9.0 (compatible; Staddlebot/1.0)":                                                                                                                                                                                    true,
 	"myspider/Nutch-1.10": true,
+	"navigation":          true,
 	"netEstate NE Crawler (+http://www.website-datenbank.de/)":               true,
+	"Netvibes (crawler; http://www.netvibes.com)":                            true,
+	"Netvibes (crawler; https://www.netvibes.com)":                           true,
+	"Nextcloud Server Crawler":                                               true,
 	"omgili/0.5 +http://omgili.com":                                          true,
+	"OWLer/0.1 (built with StormCrawler; https://ows.eu/owler; owl@ow-s.eu)": true,
 	"PercolateCrawler/4 (ops@percolate.com)":                                 true,
-	"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-43-generic": true,
+	"PhxBot/0.1 (phxbot@protonmail.com)":                                     true,
+	"PlayStore-Google Mozilla/5.0 Chrome/114.0.5735.179  (KHTML, like Gecko; compatible; +http://www.google.com/bot.html)":                                        true,
+	"PlurkBot; Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0(facebookexternalhitTwitterbot compatible; PlurkBot; +https://www.plurk.com/)": true,
+	"PubMatic Crawler Bot":                                        true,
+	"PulsePoint-Ads.txt-Crawler/1.1":                              true,
 	"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true,
-	"Python-urllib/2.7":                                           true,
+	"Quora-Bot/1.0 (http://www.quora.com)":                        true,
+	"R6_CommentReader(www.radian6.com/crawler)":                   true,
+	"R6_FeedFetcher(www.radian6.com/crawler)":                     true,
+	"RandomWebSiteBot 1.0":                                        true,
+	"read.write.as":                                               true,
+	"Readybot.io (https://readybot.io)":                           true,
+	"repology-linkchecker/1 (+https://repology.org/docs/bots)":    true,
 	"rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true,
 	"Ruby": true,
-	"SafeDNSBot (https://www.safedns.com/searchbot)":                                                                                                                            true,
 	"SafeDNS search bot/Nutch-1.9 (https://www.safedns.com/searchbot; support [at] safedns [dot] com)": true,
-	"SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
+	"SafeDNSBot (https://www.safedns.com/searchbot)":                                                   true,
-	"Screaming Frog SEO Spider/4.1":                                           true,
+	"Sellers.Guide Crawler by Primis":                                                                  true,
-	"Screaming Frog SEO Spider/5.0":                                           true,
-	"Screaming Frog SEO Spider/5.1":                                           true,
 	"semanticbot":                                                                                      true,
-	"semanticbot (info@semanticaudience.com)":                                 true,
+	"SEMrushBot":                                                                                       true,
+	"SeRanking SEOChecker":                                                                             true,
+	"SerendeputyBot/0.8.6 (http://serendeputy.com/about/serendeputy-bot)":                              true,
+	"serpstatbot/2.1 (advanced backlink tracking bot; https://serpstatbot.com/; abuse@serpstatbot.com)": true,
 	"ShowyouBot (http://showyou.com/crawler)":                                                           true,
+	"SiteCheckerBotCrawler/1.0 (+http://sitechecker.pro)":                                               true,
+	"Slack-ImgProxy (+https://api.slack.com/robots)":                                                    true,
 	"Slackbot 1.0 (+https://api.slack.com/robots)":                                                      true,
 	"Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)":                                        true,
+	"Slzji.com Search Bot (https://www.slzji.com/search_bot)":                                           true,
+	"Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots":                       true,
+	"Snap-URL-Preview (bot; snapchat; +https://developers.snap.com/robots)":                             true,
 	"Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)":                           true,
 	"spider":          true,
 	"spiderbot":       true,
 	"Stratagems Kumo": true,
+	"Superfeedr bot/2.0 http://superfeedr.com - Make your feeds realtime: get in touch - feed-id:1396250262": true,
+	"Synapse (bot; +https://github.com/matrix-org/synapse)":                                                  true,
+	"TelegramBot (like TwitterBot)":                                                                          true,
+	"TheFeedReaderBot/2.0":                                                                                   true,
+	"tiny.write.as":                                                                                          true,
 	"Traackr.com Bot":                                                                                        true,
+	"Twingly Recon":                                                                                          true,
+	"Twitterbot":                                                                                             true,
 	"voltron":                                                                                                true,
+	"webprosbot/2.0 (+mailto:abuse-6337@webpros.com)":                                                        true,
+	"webscraper":                                                                                             true,
+	"Who.is Bot":                                                                                             true,
 	"Wotbox/2.01 (+http://www.wotbox.com/bot/)":                                                              true,
-	"yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true,
+	"wp.com feedbot/1.0 (+https://wp.com)":                                                                   true,
-	"yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html":                  true,
+	"Write.as v1.7.0; Android":                                                                               true,
-	"YisouSpider": true,
+	"write.as":                                                                                               true,
+	"WriteFreely.org Crawler (https://writefreely.org/instances)":                         true,
 	"Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true,
+	"YisouSpider": true,
+	"ZoominfoBot (zoominfobot at zoominfo dot com)": true,
 }
 
 var botPrefixes = []string{
+	"AdsTxtCrawler/",
+	"Akkoma ",
+	"Apache-HttpClient/",
+	"bitlybot/",
+	"curl/",
+	"DuckDuckBot/",
+	"facebookexternalhit/",
+	"FediDB/",
 	"Friendica",
+	"Googlebot-Image/",
+	"Googlebot/",
+	"hackney/",
+	"http.rb/",
+	"kbinBot ",
+	"libwww-perl/",
+	"Mediumbot-MetaTagFetcher/",
 	"Mozilla/5.0 (compatible; AhrefsBot/",
 	"Mozilla/5.0 (compatible; Applebot/",
 	"Mozilla/5.0 (compatible; archive.org_bot",
 	"PHP/",
-	"TelegramBot",
+	"PixelFedBot/",
-	"Twitterbot/",
+	"Pleroma ",
-	"hackney/",
-	"http.rb/",
 	"python-requests/",
+	"Python-urllib/",
+	"RepoLookoutBot/",
+	"Screaming Frog SEO Spider/",
+	"semanticbot ",
+	"SummalyBot/",
+	"TelegramBot",
+	"Twingly Recon-",
+	"Twitterbot/",
+	"WhatsApp/",
+	"yacybot ",
+}
+
+var botPhrases = []string{
+	"bingbot",
+	"Googlebot",
+	"archive.org_bot",
 }
 
 // IsBot returns whether or not the provided User-Agent string is a known bot
@@ -236,5 +439,10 @@ func IsBot(ua string) bool {
 			return true
 		}
 	}
+	for _, p := range botPhrases {
+		if strings.Contains(ua, p) {
+			return true
+		}
+	}
 	return false
 }

bots/bots_test.go

@@ -13,6 +13,8 @@ func TestIsBot(t *testing.T) {
 		"Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)":                 true,
 		"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true,
 		"Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)":                        true,
+
+		"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
 	}
 
 	for ua, r := range tests {

category/category.go

@@ -17,6 +17,11 @@ type Category struct {
 	Slug      string `json:"slug"`
 	Title     string `json:"title"`
 	PostCount int64  `json:"post_count"`
+
+	// IsCategory distinguishes this Category from a mere tag. If true, it is often prominently featured on a blog,
+	// and looked up via its Slug instead of its Hashtag. It usually has all metadata, such as Title, correctly
+	// populated.
+	IsCategory bool `json:"-"`
 }
 
 // NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the

go.mod

@@ -1,19 +1,24 @@
 module github.com/writeas/web-core
 
-go 1.10
+go 1.25.0
 
 require (
 	github.com/gofrs/uuid v3.3.0+incompatible
 	github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec
-	github.com/microcosm-cc/bluemonday v1.0.5
+	github.com/microcosm-cc/bluemonday v1.0.23
-	github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect
+	github.com/writeas/go-strip-markdown/v2 v2.1.1
-	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
-	github.com/writeas/go-strip-markdown v2.0.1+incompatible
 	github.com/writeas/impart v1.1.1
 	github.com/writeas/openssl-go v1.0.0
 	github.com/writeas/saturday v1.7.1
 	github.com/writeas/slug v1.2.0
-	golang.org/x/crypto v0.0.0-20200109152110-61a87790db17
+	golang.org/x/crypto v0.52.0
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+)
+
+require (
+	github.com/aymerick/douceur v0.2.0 // indirect
+	github.com/gorilla/css v1.0.0 // indirect
+	github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	golang.org/x/net v0.55.0 // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 // indirect
 )

go.sum

@@ -1,26 +1,19 @@
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
-github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU=
-github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE=
 github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec h1:ZXWuspqypleMuJy4bzYEqlMhJnGAYpLrWe5p7W3CdvI=
 github.com/kylemcc/twitter-text-go v0.0.0-20180726194232-7f582f6736ec/go.mod h1:voECJzdraJmolzPBgL9Z7ANwXf4oMXaTCsIkdiPpR/g=
-github.com/microcosm-cc/bluemonday v1.0.5 h1:cF59UCKMmmUgqN1baLvqU/B1ZsMori+duLVTLpgiG3w=
+github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
-github.com/microcosm-cc/bluemonday v1.0.5/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
+github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHVHGom5hKW5VXNc2xZIkfCKP8iaqOyYtUQ=
 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/writeas/go-strip-markdown v2.0.1+incompatible h1:IIqxTM5Jr7RzhigcL6FkrCNfXkvbR+Nbu1ls48pXYcw=
+github.com/writeas/go-strip-markdown/v2 v2.1.1 h1:hAxUM21Uhznf/FnbVGiJciqzska6iLei22Ijc3q2e28=
-github.com/writeas/go-strip-markdown v2.0.1+incompatible/go.mod h1:Rsyu10ZhbEK9pXdk8V6MVnZmTzRG0alMNLMwa0J01fE=
+github.com/writeas/go-strip-markdown/v2 v2.1.1/go.mod h1:UvvgPJgn1vvN8nWuE5e7v/+qmDu3BSVnKAB6Gl7hFzA=
 github.com/writeas/impart v1.1.1 h1:RyA9+CqbdbDuz53k+nXCWUY+NlEkdyw6+nWanxSBl5o=
 github.com/writeas/impart v1.1.1/go.mod h1:g0MpxdnTOHHrl+Ca/2oMXUHJ0PcRAEWtkCzYCJUXC9Y=
 github.com/writeas/openssl-go v1.0.0 h1:YXM1tDXeYOlTyJjoMlYLQH1xOloUimSR1WMF8kjFc5o=
@@ -29,16 +22,9 @@ github.com/writeas/saturday v1.7.1 h1:lYo1EH6CYyrFObQoA9RNWHVlpZA5iYL5Opxo7PYAnZ
 github.com/writeas/saturday v1.7.1/go.mod h1:ETE1EK6ogxptJpAgUbcJD0prAtX48bSloie80+tvnzQ=
 github.com/writeas/slug v1.2.0 h1:EMQ+cwLiOcA6EtFwUgyw3Ge18x9uflUnOnR6bp/J+/g=
 github.com/writeas/slug v1.2.0/go.mod h1:RE8shOqQP3YhsfsQe0L3RnuejfQ4Mk+JjY5YJQFubfQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
-golang.org/x/crypto v0.0.0-20200109152110-61a87790db17 h1:nVJ3guKA9qdkEQ3TUdXI9QSINo2CUPM/cySEvw2w8I0=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
-golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=

l10n/phrases_ko.go

@@ -0,0 +1,34 @@
+package l10n
+
+// Temporal korean localation by Hoto Ras (@ras@hoto.moe), 2024
+var phrasesKO = map[string]string{
+	"Anonymous post":                 "모두의 게시물",
+	"Blogs":                          "블로그",
+	"Enter":                          "접속",
+	"Newer":                          "신규",
+	"Next":                           "다음",
+	"Older":                          "오래됨",
+	"Posts":                          "게시물",
+	"Previous":                       "이전",
+	"Publish to...":                  "다음으로 게시...",
+	"Publish":                        "게시",
+	"Read more...":                   "더 읽어보기...",
+	"Subscribe":                      "구독",
+	"This blog requires a password.": "이 블로그는 비밀번호가 필요합니다.",
+	"Toggle theme":                   "테마 토글",
+	"View posts":                     "게시물 보기",
+	"delete":                         "삭제",
+	"edit":                           "수정",
+	"email subscription confirm":     "이메일을 확인하고 확인 링크를 눌러 구독을 완료하세요.",
+	"email subscription prompt":      "새로운 소식을 보고 싶으신가요? 여기에 이메일을 입력해 구독하세요!",
+	"email subscription success":     "구독이 완료되었습니다! 이제 앞으로의 블로그 게시물을 이메일로 받아볼 수 있어요.",
+	"move to...":                     "다음으로 이동...",
+	"pin":                            "고정",
+	"published with write.as":        "write.as를 통해 배포됨",
+	"share modal ending":             "친구에게 보내거나, 인터넷에 공유하거나, 트윗할 수도 있습니다. 더 알아보기",
+	"share modal introduction":       "각각의 게시물에는 비밀이 있어요. 각자 특별한 주소가 있어 모두와 공유할 수 있답니다. 주소는 여기 있어요:",
+	"share modal title":              "이 게시물 공유하기",
+	"share":                          "공유",
+	"unpin":                          "고정 해제",
+	"title dash":                     "—",
+}

l10n/strings.go

@@ -34,6 +34,8 @@ func Strings(lang string) map[string]string {
 		return phrasesIT
 	case "ja":
 		return phrasesJA
+	case "ko":
+		return phrasesKO
 	case "lt":
 		return phrasesLT
 	case "mk":

posts/parse.go

@@ -2,7 +2,8 @@ package posts
 
 import (
 	"fmt"
-	stripmd "github.com/writeas/go-strip-markdown"
+	stripmd "github.com/writeas/go-strip-markdown/v2"
+	"github.com/writeas/slug"
 	"github.com/writeas/web-core/stringmanip"
 	"regexp"
 	"strings"
@@ -37,6 +38,20 @@ func ExtractTitle(content string) (title string, body string) {
 	return
 }
 
+func PostTitle(content, friendlyId string) string {
+	content = StripHTMLWithoutEscaping(content)
+
+	content = strings.TrimLeftFunc(stripmd.Strip(content), unicode.IsSpace)
+	eol := strings.IndexRune(content, '\n')
+	blankLine := strings.Index(content, "\n\n")
+	if blankLine != -1 && blankLine <= eol && blankLine <= assumedTitleLen {
+		return strings.TrimSpace(content[:blankLine])
+	} else if utf8.RuneCountInString(content) <= maxTitleLen {
+		return content
+	}
+	return friendlyId
+}
+
 func FriendlyPostTitle(content, friendlyId string) string {
 	content = StripHTMLWithoutEscaping(content)
 
@@ -61,6 +76,7 @@ func FriendlyPostTitle(content, friendlyId string) string {
 // choosing what to generate. In case a post has a title, this function will
 // fail, and logic should instead be implemented to skip this when there's no
 // title, like so:
+//
 //	var desc string
 //	if title == "" {
 //	    desc = PostDescription(content, title, friendlyId)
@@ -163,3 +179,32 @@ func PostLede(t string, includePunc bool) string {
 
 	return t
 }
+
+func GetSlug(title, lang string) string {
+	return GetSlugFromPost("", title, lang)
+}
+
+func GetSlugFromPost(title, body, lang string) string {
+	if title == "" {
+		// Remove Markdown, so e.g. link URLs and image alt text don't make it into the slug
+		body = strings.TrimSpace(stripmd.StripOptions(body, stripmd.Options{SkipImages: true}))
+		title = PostTitle(body, body)
+	}
+	title = PostLede(title, false)
+	// Truncate lede if needed
+	title, _ = TruncToWord(title, maxTitleLen)
+	var s string
+	if lang != "" && len(lang) == 2 {
+		s = slug.MakeLang(title, lang)
+	} else {
+		s = slug.Make(title)
+	}
+
+	// Transliteration may cause the slug to expand past the limit, so truncate again
+	s, _ = TruncToWord(s, maxTitleLen)
+	return strings.TrimFunc(s, func(r rune) bool {
+		// TruncToWord doesn't respect words in a slug, since spaces are replaced
+		// with hyphens. So remove any trailing hyphens.
+		return r == '-'
+	})
+}

text/extract.go

@@ -0,0 +1,33 @@
+package text
+
+import (
+	"github.com/kylemcc/twitter-text-go/extract"
+	"net/url"
+	"regexp"
+)
+
+var imageURLRegex = regexp.MustCompile(`(?i)[^ ]+\.(gif|png|jpg|jpeg|image)$`)
+
+func ExtractImages(content string) []string {
+	matches := extract.ExtractUrls(content)
+	urls := map[string]bool{}
+	for i := range matches {
+		uRaw := matches[i].Text
+		// Parse the extracted text so we can examine the path
+		u, err := url.Parse(uRaw)
+		if err != nil {
+			continue
+		}
+		// Ensure the path looks like it leads to an image file
+		if !imageURLRegex.MatchString(u.Path) {
+			continue
+		}
+		urls[uRaw] = true
+	}
+
+	resURLs := make([]string, 0)
+	for k := range urls {
+		resURLs = append(resURLs, k)
+	}
+	return resURLs
+}