Skip to content

Commit 4d3b3c6

Browse files
kaki-adminkaki-admin
committed
log
1 parent ed5a4af commit 4d3b3c6

25 files changed

Lines changed: 293 additions & 328 deletions

‎templates/a16z.go‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package templates
22

33
import (
44
"encoding/json"
5+
"log"
56
"strings"
67
"time"
78

89
"github.com/PuerkitoBio/goquery"
9-
"recommend.common/logger"
1010
)
1111

1212
type A16ZMetaDataWithAuthorName struct {
@@ -34,7 +34,7 @@ type A16ZMetaDataWithAuthorName struct {
3434
Name string `json:"name"`
3535
Logo string `json:"logo"`
3636
} `json:"publisher"`
37-
Keywords []any `json:"keywords"`
37+
Keywords []any `json:"keywords"`
3838
DateCreated string `json:"dateCreated"`
3939
DatePublished string `json:"datePublished"`
4040
DateModified string `json:"dateModified"`
@@ -51,7 +51,7 @@ type A16ZMetadataWithAuthorID struct {
5151
Author []struct {
5252
ID string `json:"@id"`
5353
} `json:"author,omitempty"`
54-
Headline string `json:"headline,omitempty"`
54+
Headline string `json:"headline,omitempty"`
5555
DatePublished string `json:"datePublished,omitempty"`
5656
DateModified string `json:"dateModified,omitempty"`
5757
MainEntityOfPage struct {
@@ -122,7 +122,7 @@ func (t *Template) A16ZScrapMetaData(document *goquery.Document) (string, string
122122
var firstTypeMetaData A16ZMetaDataWithAuthorName
123123
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
124124
if unmarshalErr != nil {
125-
logger.Info("convert a16metadata unmarshalError %v", unmarshalErr)
125+
log.Printf("convert a16metadata unmarshalError %v", unmarshalErr)
126126
} else {
127127
for _, currentAuthro := range firstTypeMetaData.Author {
128128
if len(currentAuthro.Name) != 0 {
@@ -145,7 +145,7 @@ func (t *Template) A16ZScrapMetaData(document *goquery.Document) (string, string
145145
if len(author) == 0 {
146146
author = "a16z editorial"
147147
}
148-
logger.Info("author last: %s", author)
148+
log.Printf("author last: %s", author)
149149
return author, published_at
150150
}
151151

@@ -172,14 +172,14 @@ func (t *Template) A16ZPublishedAtTimeFromScriptMetadata(document *goquery.Docum
172172
var firstTypeMetaData A16ZMetaDataWithAuthorName
173173
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
174174
if unmarshalErr != nil {
175-
logger.Info("convert a16zmetadata unmarshalError %v", unmarshalErr)
175+
log.Printf("convert a16zmetadata unmarshalError %v", unmarshalErr)
176176
return
177177

178178
} else {
179-
currentParsedPublishedAt,parsePublishedErr := ConvertToTimestampA16Z(firstTypeMetaData.DatePublished)
179+
currentParsedPublishedAt, parsePublishedErr := ConvertToTimestampA16Z(firstTypeMetaData.DatePublished)
180180
if parsePublishedErr != nil {
181-
logger.Error("convert time fail")
182-
return;
181+
log.Printf("convert time fail")
182+
return
183183
}
184184
publishedAt = currentParsedPublishedAt
185185
// publishedAt = firstTypeMetaData.DateCreated.Unix()
@@ -192,18 +192,18 @@ func (t *Template) A16ZPublishedAtTimeFromScriptMetadata(document *goquery.Docum
192192
var secondTypeMetaData A16ZMetadataWithAuthorID
193193
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
194194
if unmarshalErr != nil {
195-
logger.Info("convert a16zmetadata unmarshalError %v", unmarshalErr)
195+
log.Printf("convert a16zmetadata unmarshalError %v", unmarshalErr)
196196
return
197197

198198
} else {
199-
for _,currentGraph := range secondTypeMetaData.Graph {
199+
for _, currentGraph := range secondTypeMetaData.Graph {
200200
if publishedAt != 0 {
201201
break
202202
}
203-
currentParsedPublishedAt,parsePublishedErr := ConvertToTimestampA16Z(currentGraph.DatePublished)
203+
currentParsedPublishedAt, parsePublishedErr := ConvertToTimestampA16Z(currentGraph.DatePublished)
204204
if parsePublishedErr != nil {
205-
logger.Error("convert time fail")
206-
return;
205+
log.Printf("convert time fail")
206+
return
207207
}
208208
publishedAt = currentParsedPublishedAt
209209
}
@@ -216,12 +216,11 @@ func (t *Template) A16ZPublishedAtTimeFromScriptMetadata(document *goquery.Docum
216216
return publishedAt
217217
}
218218

219-
220219
func ConvertToTimestampA16Z(timeStr string) (int64, error) {
221220
t, err := time.Parse(time.RFC3339, timeStr)
222221
if err != nil {
223-
return 0, err
222+
return 0, err
224223
}
225224

226225
return t.Unix(), nil
227-
}
226+
}

‎templates/bbc.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package templates
22

33
import (
44
"encoding/json"
5+
"log"
56
"strings"
67
"time"
78

89
"github.com/PuerkitoBio/goquery"
9-
"recommend.common/logger"
1010
)
1111

1212
type BBCNewsMetadataSecond struct {
@@ -137,7 +137,7 @@ func (t *Template) BBCNewsScrapMetaData(document *goquery.Document) (string, str
137137
var firstTypeMetaData BBCNewsMetaData
138138
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
139139
if unmarshalErr != nil {
140-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
140+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
141141

142142
} else {
143143
for _, currentAuthor := range firstTypeMetaData.Author {
@@ -157,7 +157,7 @@ func (t *Template) BBCNewsScrapMetaData(document *goquery.Document) (string, str
157157
var secondTypeMetaData BBCNewsMetadataSecond
158158
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
159159
if unmarshalErr != nil {
160-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
160+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
161161
return
162162
}
163163
author = secondTypeMetaData.Author.Name
@@ -191,10 +191,9 @@ func (t *Template) BBCNewsPublishedAtTimeFromScriptMetadata(document *goquery.Do
191191
var firstTypeMetaData BBCNewsMetaData
192192
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
193193
if unmarshalErr != nil {
194-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
195-
194+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
196195

197-
}else{
196+
} else {
198197
publishedAt = firstTypeMetaData.DatePublished.Unix()
199198
}
200199

@@ -204,7 +203,7 @@ func (t *Template) BBCNewsPublishedAtTimeFromScriptMetadata(document *goquery.Do
204203
var secondTypeMetaData BBCNewsMetadataSecond
205204
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
206205
if unmarshalErr != nil {
207-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
206+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
208207
return
209208
}
210209
publishedAt = secondTypeMetaData.DatePublished.Unix()

‎templates/bbcsport.go‎

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@ package templates
33
import (
44
"encoding/json"
55
"fmt"
6+
"log"
67
"strings"
78
"time"
89

910
"github.com/PuerkitoBio/goquery"
10-
"recommend.common/logger"
1111
)
1212

1313
type BBCSportMetaDataFirst struct {
@@ -74,8 +74,6 @@ type BBCSportMetaDataSecond struct {
7474
} `json:"author"`
7575
}
7676

77-
78-
7977
func (t *Template) BBCSportsScrapMetaData(document *goquery.Document) (string, string) {
8078

8179
author := ""
@@ -96,16 +94,16 @@ func (t *Template) BBCSportsScrapMetaData(document *goquery.Document) (string, s
9694
}
9795
scriptContent := strings.TrimSpace(s.Text())
9896

99-
var firstTypeMetaData BBCSportMetaDataFirst;
97+
var firstTypeMetaData BBCSportMetaDataFirst
10098
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
10199
if unmarshalErr != nil {
102-
logger.Info("convert convert bbc news unmarshalError %v",unmarshalErr)
103-
}else{
104-
for _,currentAuthor := range firstTypeMetaData.Author {
105-
if len(currentAuthor.Name) != 0{
100+
log.Printf("convert convert bbc news unmarshalError %v", unmarshalErr)
101+
} else {
102+
for _, currentAuthor := range firstTypeMetaData.Author {
103+
if len(currentAuthor.Name) != 0 {
106104
if len(author) != 0 {
107105
author = author + " & " + currentAuthor.Name
108-
}else{
106+
} else {
109107
author = currentAuthor.Name
110108
}
111109
}
@@ -114,11 +112,11 @@ func (t *Template) BBCSportsScrapMetaData(document *goquery.Document) (string, s
114112
if len(author) != 0 {
115113
return
116114
}
117-
118-
var secondTypeMetaData BBCSportMetaDataSecond;
115+
116+
var secondTypeMetaData BBCSportMetaDataSecond
119117
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
120118
if unmarshalErr != nil {
121-
logger.Error("convert BBCSport unmarshalErr %v \n %s",unmarshalErr,scriptContent)
119+
log.Printf("convert BBCSport unmarshalErr %v \n %s", unmarshalErr, scriptContent)
122120
return
123121
}
124122
author = secondTypeMetaData.Author.Name
@@ -135,17 +133,17 @@ func (t *Template) BBCSportsScrapMetaData(document *goquery.Document) (string, s
135133
if len(trimmedText) != 0 {
136134
if len(author) == 0 {
137135
author = trimmedText
138-
}else{
136+
} else {
139137
author = author + " & " + trimmedText
140138
}
141139
}
142140
})
143141
}
144-
logger.Info("author last: %s",author)
142+
log.Printf("author last: %s", author)
145143
return author, published_at
146144
}
147145

148-
func (t* Template) BBCSportsPublishedAtTimeFromScriptMetadata(document *goquery.Document) int64 {
146+
func (t *Template) BBCSportsPublishedAtTimeFromScriptMetadata(document *goquery.Document) int64 {
149147

150148
var publishedAt int64 = 0
151149

@@ -165,22 +163,22 @@ func (t* Template) BBCSportsPublishedAtTimeFromScriptMetadata(document *goquery.
165163
}
166164

167165
scriptContent := strings.TrimSpace(s.Text())
168-
var firstTypeMetaData BBCSportMetaDataFirst;
166+
var firstTypeMetaData BBCSportMetaDataFirst
169167
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
170168
if unmarshalErr != nil {
171-
logger.Info("convert SkyNewsScrap unmarshalError %v",unmarshalErr)
172-
}else{
169+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
170+
} else {
173171
publishedAt = firstTypeMetaData.DatePublished.Unix()
174172
}
175-
173+
176174
if publishedAt != 0 {
177175
return
178176
}
179177

180-
var secondTypeMetaData BBCSportMetaDataSecond;
178+
var secondTypeMetaData BBCSportMetaDataSecond
181179
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
182180
if unmarshalErr != nil {
183-
logger.Error("convert BBCSport unmarshalErr %v",unmarshalErr)
181+
log.Printf("convert BBCSport unmarshalErr %v", unmarshalErr)
184182
return
185183
}
186184
publishedAt = secondTypeMetaData.DatePublished.Unix()
@@ -193,7 +191,7 @@ func (t* Template) BBCSportsPublishedAtTimeFromScriptMetadata(document *goquery.
193191
if datetime, exists := timeTag.Attr("datetime"); exists {
194192
parsedTime, err := time.Parse(time.RFC3339, datetime)
195193
if err != nil {
196-
logger.Error("Error parsing datetime: %v", err)
194+
log.Printf("Error parsing datetime: %v", err)
197195
}
198196
timestamp := parsedTime.Unix()
199197
publishedAt = timestamp
@@ -203,4 +201,3 @@ func (t* Template) BBCSportsPublishedAtTimeFromScriptMetadata(document *goquery.
203201
}
204202
return publishedAt
205203
}
206-

‎templates/businessinsider.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package templates
22

33
import (
44
"encoding/json"
5+
"log"
56
"strings"
67
"time"
78

89
"github.com/PuerkitoBio/goquery"
9-
"recommend.common/logger"
1010
)
1111

1212
type BusinessInsiderMetaData struct {
@@ -160,7 +160,7 @@ func (t *Template) BusinessinsiderScrapMetaData(document *goquery.Document) (str
160160
var firstTypeMetaData BusinessInsiderMetaData
161161
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
162162
if unmarshalErr != nil {
163-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
163+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
164164

165165
} else {
166166
author = firstTypeMetaData.Author.Name
@@ -171,7 +171,7 @@ func (t *Template) BusinessinsiderScrapMetaData(document *goquery.Document) (str
171171
var secondTypeMetaData BusinessInsiderMetaDataSecond
172172
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
173173
if unmarshalErr != nil {
174-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
174+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
175175

176176
} else {
177177
for _, currentAuthor := range secondTypeMetaData.Author {
@@ -190,7 +190,7 @@ func (t *Template) BusinessinsiderScrapMetaData(document *goquery.Document) (str
190190
break
191191
}
192192
}
193-
logger.Info("author last: %s", author)
193+
log.Printf("author last: %s", author)
194194
return author, published_at
195195
}
196196

@@ -217,7 +217,7 @@ func (t *Template) BusinessinsiderPublishedAtTimeFromScriptMetadata(document *go
217217
var firstTypeMetaData BusinessInsiderMetaData
218218
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
219219
if unmarshalErr != nil {
220-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
220+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
221221

222222
} else {
223223
publishedAt = firstTypeMetaData.DatePublished.Unix()
@@ -228,7 +228,7 @@ func (t *Template) BusinessinsiderPublishedAtTimeFromScriptMetadata(document *go
228228
var secondTypeMetaData BusinessInsiderMetaDataSecond
229229
unmarshalErr = json.Unmarshal([]byte(scriptContent), &secondTypeMetaData)
230230
if unmarshalErr != nil {
231-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
231+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
232232

233233
} else {
234234
publishedAt = secondTypeMetaData.DatePublished.Unix()

‎templates/businesslive.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package templates
22

33
import (
44
"encoding/json"
5+
"log"
56
"strings"
67
"time"
78

89
"github.com/PuerkitoBio/goquery"
9-
"recommend.common/logger"
1010
)
1111

1212
type BusinessliveMetadata struct {
@@ -63,7 +63,7 @@ func (t *Template) BusinessliveScrapMetaData(document *goquery.Document) (string
6363
var firstTypeMetaData BusinessliveMetadata
6464
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
6565
if unmarshalErr != nil {
66-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
66+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
6767
return
6868
}
6969
author = firstTypeMetaData.Author.Name
@@ -72,7 +72,7 @@ func (t *Template) BusinessliveScrapMetaData(document *goquery.Document) (string
7272
break
7373
}
7474
}
75-
logger.Info("author last: %s", author)
75+
log.Printf("author last: %s", author)
7676
return author, published_at
7777
}
7878

@@ -99,7 +99,7 @@ func (t *Template) BusinesslivePublishedAtTimeFromScriptMetadata(document *goque
9999
var firstTypeMetaData BusinessliveMetadata
100100
unmarshalErr := json.Unmarshal([]byte(scriptContent), &firstTypeMetaData)
101101
if unmarshalErr != nil {
102-
logger.Info("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
102+
log.Printf("convert SkyNewsScrap unmarshalError %v", unmarshalErr)
103103
return
104104

105105
}

0 commit comments

Comments
 (0)