From 06e32c334034b87e67ca8e9bf65eccb33746ccd9 Mon Sep 17 00:00:00 2001
From: Alexandre Morignot <erdnaxeli@cervoi.se>
Date: Wed, 5 Oct 2016 00:43:05 +0200
Subject: [PATCH] Site
---
bot/bot.go | 34 +++++++++++---
bot/collection.go | 29 +++++++++---
bot/factory.go | 16 ++++---
bot/post.go | 4 +-
main.go | 12 ++++-
{bot => site}/content.go | 2 +-
site/reader.go | 5 ++
site/utils.go | 40 ++++++++++++++++
site/youtube.go | 54 ++++++++++++++++++++++
tests/bot_test.go | 87 +++++++++++++++++++++++++++++------
tests/dummy.go | 5 +-
tests/mock.go | 28 +++++++++--
tests/site_test.go | 50 ++++++++++++++++++++
transport/irc/.events.go.swp | Bin 0 -> 12288 bytes
transport/irc/.print.go.swp | Bin 0 -> 12288 bytes
transport/irc/events.go | 8 ++--
transport/irc/print.go | 6 +--
17 files changed, 329 insertions(+), 51 deletions(-)
rename {bot => site}/content.go (96%)
create mode 100644 site/reader.go
create mode 100644 site/utils.go
create mode 100644 site/youtube.go
create mode 100644 tests/site_test.go
create mode 100644 transport/irc/.events.go.swp
create mode 100644 transport/irc/.print.go.swp
diff --git a/bot/bot.go b/bot/bot.go
index 7d55774..c378019 100644
--- a/bot/bot.go
+++ b/bot/bot.go
@@ -2,31 +2,53 @@ package bot
import (
"log"
+ "regexp"
"sync"
"github.com/mvdan/xurls"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
type Bot interface {
Collection
- ParseLine(author string, line string, contents chan Content) error
+ ParseLine(author string, line string, contents chan *site.Content) error
}
type PlayBot struct {
Collection
}
-func NewPlayBot(source string, db Db) *PlayBot {
+func NewPlayBot(source string, db Db, readers []site.Reader) *PlayBot {
return &PlayBot{
Collection: &PlayBotCollection{
- Source: source,
- Db: db,
+ Source: source,
+ Db: db,
+ Readers: readers,
},
}
}
-func (pb *PlayBot) ParseLine(author string, line string, contents chan Content) error {
+func (*PlayBot) ExtractTags(line string) []string {
+ tagsSeen := make(map[string]bool)
+ var tags []string
+ re := regexp.MustCompile(`(?:^| )#([a-zA-Z0-9_]+)`)
+
+ for _, match := range re.FindAllStringSubmatch(line, -1) {
+ if tagsSeen[match[1]] {
+ continue
+ }
+
+ tagsSeen[match[1]] = true
+ tags = append(tags, match[1])
+ }
+
+ return tags
+}
+
+func (pb *PlayBot) ParseLine(author string, line string, contents chan *site.Content) error {
+ tags := pb.ExtractTags(line)
urls := xurls.Strict.FindAllString(line, -1)
var wg sync.WaitGroup
@@ -35,7 +57,7 @@ func (pb *PlayBot) ParseLine(author string, line string, contents chan Content)
go func(url string) {
defer wg.Done()
- content, err := pb.Add(url)
+ content, err := pb.Add(url, tags)
if err != nil {
log.Print(err)
} else {
diff --git a/bot/collection.go b/bot/collection.go
index 38d09d6..49d22cc 100644
--- a/bot/collection.go
+++ b/bot/collection.go
@@ -2,23 +2,38 @@ package bot
import (
"log"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
type Collection interface {
- Add(url string) (Content, error)
+ Add(url string, tags []string) (*site.Content, error)
Get()
}
type PlayBotCollection struct {
- Source string
- Db Db
+ Source string
+ Db Db
+ Readers []site.Reader
}
-func (pb *PlayBotCollection) Add(url string) (Content, error) {
- log.Printf("%s: Add() %s", pb.Source, url)
- content := Content{Title: url}
+func (pb *PlayBotCollection) Add(url string, tags []string) (*site.Content, error) {
+ log.Printf("%s: Add() %s, %q", pb.Source, url, tags)
+
+ var content *site.Content
+ for _, reader := range pb.Readers {
+ var err error
+ content, err = reader.Read(url)
+ if err != nil {
+ return nil, err
+ }
+ if content != nil {
+ break
+ }
+ }
+
pb.Db.Create(content)
- return Content{}, nil
+ return content, nil
}
func (*PlayBotCollection) Get() {
diff --git a/bot/factory.go b/bot/factory.go
index dbdf0c6..9db1936 100644
--- a/bot/factory.go
+++ b/bot/factory.go
@@ -2,28 +2,32 @@ package bot
import (
"log"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
type Factory interface {
GetBot(string) Bot
}
-type playBotFactory struct{}
+type playBotFactory struct {
+ readers []site.Reader
+}
-func NewPlayBotFactory(dbParams DbParams) *playBotFactory {
- return &playBotFactory{}
+func NewPlayBotFactory(dbParams DbParams, readers []site.Reader) *playBotFactory {
+ return &playBotFactory{readers}
}
-func (*playBotFactory) GetBot(source string) Bot {
+func (f *playBotFactory) GetBot(source string) Bot {
// give DBÂ as arg
db := &DummyDb{}
- return NewPlayBot(source, db)
+ return NewPlayBot(source, db, f.readers)
}
type DummyDb struct{}
func (db *DummyDb) Create(value interface{}) Db {
- log.Printf("Create():Â %s", value)
+ log.Printf("Create():Â %q", value)
return db
}
diff --git a/bot/post.go b/bot/post.go
index 096ad4c..0a37648 100644
--- a/bot/post.go
+++ b/bot/post.go
@@ -2,6 +2,8 @@ package bot
import (
"time"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
type Post struct {
@@ -9,7 +11,7 @@ type Post struct {
Date time.Time
Author string `gorm:"column:sender_irc"`
Source string `gorm:"column:chan"`
- Content Content
+ Content site.Content
Tag Tag
}
diff --git a/main.go b/main.go
index 2cd35ab..a7368b0 100644
--- a/main.go
+++ b/main.go
@@ -8,6 +8,7 @@ import (
"gopkg.in/yaml.v2"
"git.iiens.net/morignot2011/playbot/bot"
+ "git.iiens.net/morignot2011/playbot/site"
"git.iiens.net/morignot2011/playbot/transport/irc"
)
@@ -54,7 +55,16 @@ func main() {
var config config
err = yaml.Unmarshal(file, &config)
quit := make(map[string](chan bool))
- factory := bot.NewPlayBotFactory(config.Db)
+
+ youtube, err := site.NewYoutube("AIzaSyD32EU9lb9cnNcEaSUBKCkKodk6rjl3HYc")
+ if err != nil {
+ log.Fatal(err)
+ }
+ readers := []site.Reader{
+ //&site.Youtube{},
+ youtube,
+ }
+ factory := bot.NewPlayBotFactory(config.Db, readers)
for name, config := range config.Transport {
c := startTransport(name, config, factory)
diff --git a/bot/content.go b/site/content.go
similarity index 96%
rename from bot/content.go
rename to site/content.go
index 18a44ce..96109b3 100644
--- a/bot/content.go
+++ b/site/content.go
@@ -1,4 +1,4 @@
-package bot
+package site
type Content struct {
Author string `gorm:"column:sender"`
diff --git a/site/reader.go b/site/reader.go
new file mode 100644
index 0000000..f7c4260
--- /dev/null
+++ b/site/reader.go
@@ -0,0 +1,5 @@
+package site
+
+type Reader interface {
+ Read(url string) (*Content, error)
+}
diff --git a/site/utils.go b/site/utils.go
new file mode 100644
index 0000000..aee25b1
--- /dev/null
+++ b/site/utils.go
@@ -0,0 +1,40 @@
+package site
+
+import (
+ "fmt"
+ "regexp"
+ "strconv"
+)
+
+type NotISO8601 struct {
+ s string
+}
+
+func (e NotISO8601) Error() string {
+ return fmt.Sprintf("'%s' is not in ISO 8601 format")
+}
+
+func ParseISO8601Duration(s string) (int, error) {
+ re := regexp.MustCompile(`^PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$`)
+ match := re.FindStringSubmatch(s)
+
+ if len(match) == 0 {
+ return -1, NotISO8601{s}
+ }
+
+ var duration int
+ if match[1] != "" {
+ hours, _ := strconv.Atoi(match[1])
+ duration += 3600 * hours
+ }
+ if match[2] != "" {
+ minutes, _ := strconv.Atoi(match[2])
+ duration += 60 * minutes
+ }
+ if match[3] != "" {
+ seconds, _ := strconv.Atoi(match[3])
+ duration += seconds
+ }
+
+ return duration, nil
+}
diff --git a/site/youtube.go b/site/youtube.go
new file mode 100644
index 0000000..c872b47
--- /dev/null
+++ b/site/youtube.go
@@ -0,0 +1,54 @@
+package site
+
+import (
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "regexp"
+
+ "google.golang.org/api/googleapi/transport"
+ "google.golang.org/api/youtube/v3"
+)
+
+type Youtube struct {
+ service *youtube.Service
+}
+
+func NewYoutube(key string) (Reader, error) {
+ client := &http.Client{
+ Transport: &transport.APIKey{Key: key},
+ }
+
+ service, err := youtube.New(client)
+ return &Youtube{service: service}, err
+}
+
+func (yt Youtube) Read(url string) (*Content, error) {
+ log.Print(url)
+ re := regexp.MustCompile(`(?:^|[^!])https?://(?:www.youtube.com/watch\?[a-zA-Z0-9_=&-]*v=|youtu.be/)([a-zA-Z0-9_-]+)`)
+ match := re.FindStringSubmatch(url)
+ log.Print(match)
+ if len(match) == 0 {
+ return nil, nil
+ }
+
+ response, err := yt.service.Videos.List("snippet,contentDetails").Id(match[1]).Do()
+ if err != nil {
+ return nil, err
+ }
+
+ video := response.Items[0]
+ duration, err := ParseISO8601Duration(video.ContentDetails.Duration)
+ if err != nil {
+ return nil, errors.New(fmt.Sprintf("Invalid duration: %s", err))
+ }
+
+ return &Content{
+ Author: video.Snippet.ChannelTitle,
+ Duration: duration + 1,
+ Source: "youtube",
+ SourceId: video.Id,
+ Url: "https://www.youtube.com/watch?v=" + video.Id,
+ }, nil
+}
diff --git a/tests/bot_test.go b/tests/bot_test.go
index 2c662a7..23fadc8 100644
--- a/tests/bot_test.go
+++ b/tests/bot_test.go
@@ -1,17 +1,18 @@
package test
import (
- "reflect"
+ "reflect"
"sort"
"testing"
"git.iiens.net/morignot2011/playbot/bot"
+ "git.iiens.net/morignot2011/playbot/site"
)
func TestAdd(t *testing.T) {
db := &DummyDb{}
bot := bot.NewPlayBot("test", db)
- content, err := bot.Add("http://www.youtube.com/watch?v=IiWapK6WQPg")
+ content, err := bot.Add("http://www.youtube.com/watch?v=IiWapK6WQPg", []string{"hardstle", "raw"})
if err != nil {
t.Fatalf("Receive error %s", err)
@@ -48,43 +49,99 @@ func TestAdd(t *testing.T) {
}
}
+func TestExtractTags(t *testing.T) {
+ db := &DummyDb{}
+ bot := bot.NewPlayBot("test", db)
+
+ testCases := []struct {
+ msg string
+ tags []string
+ }{
+ {
+ msg: "hello #boy, wa#nna dance?",
+ tags: []string{"boy"},
+ },
+ {
+ msg: "#I'm #Bond, James #Bond",
+ tags: []string{"I", "Bond"},
+ },
+ {
+ msg: "last #test but not #least",
+ tags: []string{"test", "least"},
+ },
+ }
+
+ for _, c := range testCases {
+ tags := bot.ExtractTags(c.msg)
+ if !reflect.DeepEqual(tags, c.tags) {
+ t.Error("Expected %q but got %q", c.tags, tags)
+ }
+ }
+}
+
func TestParseLine(t *testing.T) {
testCases := []struct {
msg string
- expected []string
+ expected []AddArgs
}{
{
- msg: "this is a http://perdu.com test",
- expected: []string{"http://perdu.com"},
+ msg: "this is a http://perdu.com test",
+ expected: []AddArgs{
+ {
+ url: "http://perdu.com",
+ tags: nil,
+ },
+ },
},
{
msg: "nothing here",
- expected: []string{},
+ expected: nil,
+ },
+ {
+ msg: "some http://giantbatfarts.com/ #with #tags",
+ expected: []AddArgs{
+ {
+ url: "http://giantbatfarts.com/",
+ tags: []string{"tags", "with"},
+ },
+ },
},
{
- msg: "some http://url.com #with #tags",
- expected: []string{"http://url.com"},
+ msg: "https://youtu.be/zpG4CZcTpRM?t=1m24s url: http://heeeeeeeey.com/ #oki",
+ expected: []AddArgs{
+ {
+ url: "http://heeeeeeeey.com/",
+ tags: []string{"oki"},
+ },
+ {
+ url: "https://youtu.be/zpG4CZcTpRM?t=1m24s",
+ tags: []string{"oki"},
+ },
+ },
},
{
- msg: "http://two.com urls: http://second.com",
- expected: []string{"http://second.com", "http://two.com"},
+ msg: "fake tag https://translate.google.com/#en/ja/PlayBot",
+ expected: []AddArgs{
+ {
+ url: "https://translate.google.com/#en/ja/PlayBot",
+ tags: nil,
+ },
+ },
},
}
for _, c := range testCases {
- fc := &FakeCollection{
- AddCalls: []string{},
- }
+ fc := &FakeCollection{}
testBot := &bot.PlayBot{
Collection: fc,
}
- contents := make(chan bot.Content)
+ contents := make(chan site.Content)
go testBot.ParseLine("me", c.msg, contents)
for range contents {
}
- sort.Strings(fc.AddCalls)
+ sort.Sort(ByUrl(fc.AddCalls))
if !reflect.DeepEqual(fc.AddCalls, c.expected) {
t.Errorf("Expected %q but got %q", c.expected, fc.AddCalls)
}
diff --git a/tests/dummy.go b/tests/dummy.go
index 527d9fb..2bc53d5 100644
--- a/tests/dummy.go
+++ b/tests/dummy.go
@@ -2,13 +2,14 @@ package test
import (
"git.iiens.net/morignot2011/playbot/bot"
+ "git.iiens.net/morignot2011/playbot/site"
)
type DummyDb struct {
- createCalls []bot.Content
+ createCalls []site.Content
}
func (db *DummyDb) Create(value interface{}) bot.Db {
- db.createCalls = append(db.createCalls, value.(bot.Content))
+ db.createCalls = append(db.createCalls, value.(site.Content))
return db
}
diff --git a/tests/mock.go b/tests/mock.go
index 6333ad5..f044181 100644
--- a/tests/mock.go
+++ b/tests/mock.go
@@ -1,17 +1,35 @@
package test
import (
- "git.iiens.net/morignot2011/playbot/bot"
+ "sort"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
type FakeCollection struct {
- AddCalls []string
+ AddCalls []AddArgs
+}
+
+type AddArgs struct {
+ url string
+ tags []string
}
-func (c *FakeCollection) Add(url string) (bot.Content, error) {
- c.AddCalls = append(c.AddCalls, url)
+type ByUrl []AddArgs
+
+func (a ByUrl) Len() int { return len(a) }
+func (a ByUrl) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByUrl) Less(i, j int) bool { return a[i].url < a[j].url }
+
+func (c *FakeCollection) Add(url string, tags []string) (site.Content, error) {
+ sort.Strings(tags)
+
+ c.AddCalls = append(c.AddCalls, AddArgs{
+ url: url,
+ tags: tags,
+ })
- return bot.Content{}, nil
+ return site.Content{}, nil
}
func (*FakeCollection) Get() {
diff --git a/tests/site_test.go b/tests/site_test.go
new file mode 100644
index 0000000..57ea278
--- /dev/null
+++ b/tests/site_test.go
@@ -0,0 +1,50 @@
+package test
+
+import (
+ "testing"
+
+ "git.iiens.net/morignot2011/playbot/site"
+)
+
+func TestParseISO8601Duration(t *testing.T) {
+ testCases := []struct {
+ s string
+ i int
+ }{
+ {
+ s: "PT5H",
+ i: 18000,
+ },
+ {
+ s: "PT0S",
+ i: 0,
+ },
+ {
+ s: "P5H3M1S",
+ i: -1,
+ },
+ {
+ s: "PT5H3m1S",
+ i: -1,
+ },
+ {
+ s: "PT5H3M1S",
+ i: 18181,
+ },
+ {
+ s: "PT98M1234S",
+ i: 7114,
+ },
+ {
+ s: "",
+ i: -1,
+ },
+ }
+
+ for _, c := range testCases {
+ d, _ := site.ParseISO8601Duration(c.s)
+ if d != c.i {
+ t.Errorf("Expected %d but got %d for '%s'", c.i, d, c.s)
+ }
+ }
+}
diff --git a/transport/irc/.events.go.swp b/transport/irc/.events.go.swp
new file mode 100644
index 0000000000000000000000000000000000000000..92779672c8ec1df79889d67ee8382586b7a777ee
GIT binary patch
literal 12288
zcmeI2L2DC16vwB6hlo|Fg4c0H*re%fQ!hpAp$bY%m8w+_rO0M?vK_LSiL;YpTUrl_
z;9c<xc=987)DPgrFQF&FUcC9AWD`(msK?4%^3O2y-o7{QH=Dzf*PMN{bcfH2S%&Kz
zW4~$#k9STjv#*XZX;U3PlezM?G1rz#Wj@9}H0_T9Mu|z|ohUO=VUs9RPADaHA+&T+
z$5@qV<K|`?ji{T)+fCylXJcJ-jdhX24RT9rSBT6EBhyF(h`?9^J=dC<pJZ3(8vUR8
z<=_%OKlO0zEo3ADM1Tko0U|&IhyW2F0z}~dBH+3c>^Vw0QI-{y<HTQMIi-OJ5CI}U
z1c(3;AOb{y2oM1xKm>>Y5jchf;*>Fcnz5-#Y##sr58wX}&NB8B`VM`DK0)uGx6m8t
z9yAS|gnpcX9r^&hhF(E0p%>5sGy_dTHRv1ie1Sef@1cEY4|)LIhZ@jD=mO;T)Zf=J
z<Vss60z`la5CI}U1c(3;AOb|-I1t!l6?`gAbgk0vrqK?~#$hrTtE|#8mX|YrZGodP
zIg`VMg6}d`$qa9b)z}ttMQIu4_!4(j(ysGPk<~}ORk)Uc3LmKxw`G`Y#+ug(<)kQ6
zBW2T@oj@aWvF;tT%r{sC!TfqZ{G!c@^#&G8^T@Ck+f2H!%33|0@X+zvl1<iz-PBc%
zBDWC8*N41luJ&nly&rWj_cc^~U7+H`)1Gqt`V4orCl8C6s=|kpiIeFO1AF2qPb{Ws
z8001wiRlD&zP?d7oI<v{2MWmk2LZmoF(2GG0(2`*oUz;Dwsbd)3j<%#sJA@uQT*1#
zvQfIv8Yl;$S4@^kD^~rXx5B_X@GAxOP!ffgKlvx41)p+o_)e5Q{`e2!{7*oTD>y<^
zynjmYzFt9%-1~3#nx0$K%6qL=F40~OOO)hjt#<)a9p69CL&mys(vI=a!peRFroA~6
literal 0
HcmV?d00001
diff --git a/transport/irc/.print.go.swp b/transport/irc/.print.go.swp
new file mode 100644
index 0000000000000000000000000000000000000000..853d46928d8b839cbf92af3320f486514dd72286
GIT binary patch
literal 12288
zcmeI&%TC)s6b9gvN-U@fRP+Hl7$G^d*a_W4Y%3N)f`SC?+Hoe<2s7i3C%qvqZ^H|K
zcoCj}?%0w&92>ifEUYeD^pEswlFXSC%_n6Mc?O-&{cX_;I>h5O(agKam-ClfbmNGm
z9qQ<ZOjWVgFutjwOdHngrPDtR!`KX?Utwax+{R&|T%eTHd7z~WkBn7`Hg3Js?S@$z
z{Tvt<IveRcGuDNQH)K|67bK=`j2QwDXcic|(OPegKCE}kpZc=5B;G6@H1EMG1Rwwb
z2tWV=5P$##An>dNT-Ks9KGBQnR14MInwqP(m>>WF2tWV=5P$##AOHafKmY;|_zwl5
zA<@M=(F3P={$Kz9pYV%&&K>7B=azHLx#C=MzH|0CpEx_5x11KI=<|wa3<y8~0uX=z
z1Rwwb2tWV=5P-lxByd8rsY!y};zIez_cpCHc0;V>-g_afZPRG1W8pio+PCq3eFwcQ
z$_waavCm~isj+QwOf=Q6>?=}}a?h{aGd9{u?V9_3d@Ay*VzHVlCxhzK@>|+8>;1*0
d`C^|(T+2uFNF5dT=Y>y{Me$*jNWoX>=@Bb7kWc^s
literal 0
HcmV?d00001
diff --git a/transport/irc/events.go b/transport/irc/events.go
index 4292c1d..6457110 100644
--- a/transport/irc/events.go
+++ b/transport/irc/events.go
@@ -1,9 +1,9 @@
package irc
import (
- "git.iiens.net/morignot2011/playbot/bot"
-
irc "github.com/fluffle/goirc/client"
+
+ "git.iiens.net/morignot2011/playbot/site"
)
func (t *IrcTransport) connected(conn *irc.Conn, line *irc.Line) {
@@ -12,7 +12,7 @@ func (t *IrcTransport) connected(conn *irc.Conn, line *irc.Line) {
conn.Join(channel)
bot := t.botFactory.GetBot("irc.iiens.net")
t.bots[channel] = bot
- bot.Add("lol.com")
+ bot.Add("lol.com", []string{})
}
}
@@ -24,7 +24,7 @@ func (t *IrcTransport) privmsg(conn *irc.Conn, line *irc.Line) {
channel := line.Target()
msg := line.Args[1]
b := t.bots[channel]
- contents := make(chan bot.Content)
+ contents := make(chan *site.Content)
go b.ParseLine(line.Nick, msg, contents)
diff --git a/transport/irc/print.go b/transport/irc/print.go
index 2c85e53..e7cc912 100644
--- a/transport/irc/print.go
+++ b/transport/irc/print.go
@@ -3,11 +3,11 @@ package irc
import (
"log"
- "git.iiens.net/morignot2011/playbot/bot"
+ "git.iiens.net/morignot2011/playbot/site"
)
-func (t *IrcTransport) printContent(content bot.Content) {
- log.Printf("Print lol")
+func (t *IrcTransport) printContent(content *site.Content) {
+ log.Printf("Print lol: %q", content)
}
func (t *IrcTransport) printError(err error) {
--
GitLab