Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
C
crawler.vmusicchart.vn
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Phạm Văn Đoan
crawler.vmusicchart.vn
Commits
d6486fa6
Commit
d6486fa6
authored
Jul 01, 2020
by
Phạm Văn Đoan
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
cập nhật blacklist listen lấy từ db
parent
f513c13c
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
55 additions
and
9 deletions
+55
-9
CrawlerTrackListenCommand.php
app/Console/Commands/CrawlerTrackListenCommand.php
+4
-1
UpdateTrackArtistCommand.php
app/Console/Commands/Update/UpdateTrackArtistCommand.php
+1
-1
Constants.php
app/Helpers/Constants.php
+1
-0
CrawlerListenBlacklist.php
app/Models/CrawlerListenBlacklist.php
+18
-0
TrackRepository.php
app/Repositories/TrackRepository.php
+31
-7
No files found.
app/Console/Commands/CrawlerTrackListenCommand.php
View file @
d6486fa6
...
@@ -66,8 +66,11 @@ class CrawlerTrackListenCommand extends Command
...
@@ -66,8 +66,11 @@ class CrawlerTrackListenCommand extends Command
echo
"
\n
Tổng số bài hát đã lấy lượt nghe hôm nay: "
.
count
(
$crawlered
);
echo
"
\n
Tổng số bài hát đã lấy lượt nghe hôm nay: "
.
count
(
$crawlered
);
// Lấy blacklist
$blacklist_listen
=
$this
->
trackRepository
->
getBacklistListen
(
$from_page
);
// Lấy mảng bài hát theo từng hệ thống nhạc
// Lấy mảng bài hát theo từng hệ thống nhạc
$inserted
=
$this
->
trackRepository
->
getTrackBySource
(
$from_page
,
$crawlered
,
$limit
);
$inserted
=
$this
->
trackRepository
->
getTrackBySource
(
$from_page
,
$crawlered
,
$limit
,
false
,
$blacklist_listen
);
$total
=
count
(
$inserted
);
$total
=
count
(
$inserted
);
echo
"
\n
Tổng số bài hát cần lấy lượt nghe: "
.
$total
;
echo
"
\n
Tổng số bài hát cần lấy lượt nghe: "
.
$total
;
...
...
app/Console/Commands/Update/UpdateTrackArtistCommand.php
View file @
d6486fa6
...
@@ -169,7 +169,7 @@ class UpdateTrackArtistCommand extends Command
...
@@ -169,7 +169,7 @@ class UpdateTrackArtistCommand extends Command
}
}
// Xử lý cập nhật nhạc sĩ ------------------------------------------------------------------------------
// Xử lý cập nhật nhạc sĩ ------------------------------------------------------------------------------
$tracks_nct
=
$this
->
trackRepository
->
getTrackBySource
(
$src
,
null
,
100
,
true
);
$tracks_nct
=
$this
->
trackRepository
->
getTrackBySource
(
$src
,
null
,
100
,
true
,
null
);
$track_id_composer
=
$this
->
trackRepository
->
getComposerBySource
(
$src
,
$tracks_nct
);
$track_id_composer
=
$this
->
trackRepository
->
getComposerBySource
(
$src
,
$tracks_nct
);
if
(
count
(
$track_id_composer
)
>
0
)
{
if
(
count
(
$track_id_composer
)
>
0
)
{
$this
->
privateUpdateTrackSinger
(
$src
,
$tracks_nct
,
[],
$track_id_composer
);
$this
->
privateUpdateTrackSinger
(
$src
,
$tracks_nct
,
[],
$track_id_composer
);
...
...
app/Helpers/Constants.php
View file @
d6486fa6
...
@@ -6,6 +6,7 @@ class Constants
...
@@ -6,6 +6,7 @@ class Constants
{
{
const
TABLE_ARTISTS
=
'artists'
;
const
TABLE_ARTISTS
=
'artists'
;
const
TABLE_COUNT_VIEWS
=
'count_views'
;
const
TABLE_COUNT_VIEWS
=
'count_views'
;
const
TABLE_CRAWLER_LISTEN_BLACKLISTS
=
'crawler_listen_blacklists'
;
const
TABLE_CRAWLER_LISTEN_HISTORIES
=
'crawler_listen_histories'
;
const
TABLE_CRAWLER_LISTEN_HISTORIES
=
'crawler_listen_histories'
;
const
TABLE_CRAWLER_LISTEN_NCTS
=
'crawler_listen_ncts'
;
const
TABLE_CRAWLER_LISTEN_NCTS
=
'crawler_listen_ncts'
;
const
TABLE_CRAWLER_LISTENS
=
'crawler_listens'
;
const
TABLE_CRAWLER_LISTENS
=
'crawler_listens'
;
...
...
app/Models/CrawlerListenBlacklist.php
0 → 100644
View file @
d6486fa6
<?php
namespace
App\Models
;
use
App\Helpers\Constants
;
use
Illuminate\Database\Eloquent\Model
;
class
CrawlerListenBlacklist
extends
Model
{
protected
$table
=
Constants
::
TABLE_CRAWLER_LISTEN_BLACKLISTS
;
public
$timestamps
=
true
;
protected
$fillable
=
[
'track_id'
,
'src'
,
'title'
];
}
app/Repositories/TrackRepository.php
View file @
d6486fa6
...
@@ -4,6 +4,7 @@ namespace App\Repositories;
...
@@ -4,6 +4,7 @@ namespace App\Repositories;
use
App\Helpers\Constants
;
use
App\Helpers\Constants
;
use
App\Models\CrawlerListen
;
use
App\Models\CrawlerListen
;
use
App\Models\CrawlerListenBlacklist
;
use
App\Models\Track
;
use
App\Models\Track
;
use
Sunra\PhpSimple\HtmlDomParser
;
use
Sunra\PhpSimple\HtmlDomParser
;
use
Yangqi\Htmldom\Htmldom
;
use
Yangqi\Htmldom\Htmldom
;
...
@@ -292,8 +293,8 @@ class TrackRepository extends BaseRepository
...
@@ -292,8 +293,8 @@ class TrackRepository extends BaseRepository
* Lấy ds bài hát chưa lấy lượt nghe trong ngày theo từng hệ thống nhạc
* Lấy ds bài hát chưa lấy lượt nghe trong ngày theo từng hệ thống nhạc
* Bài nào đã lấy rồi thì sẽ ko lấy nữa
* Bài nào đã lấy rồi thì sẽ ko lấy nữa
* Mỗi lần quét 500 bài cho đến hết
* Mỗi lần quét 500 bài cho đến hết
* Các bài
chỉ lấy ở Zing: 1198,1700,1954,3223,3334,3357,3529,3688
* Các bài
không lấy ở Zing: 1139,3835,3664
* Các bài
chỉ lấy ở NCT: 1139,3835
* Các bài
không lấy ở NCT: 1198,1700,1954,3223,3334,3357,3529,3688,3918,3664,3910
*
*
* @param string $src
* @param string $src
* @param $crawlered
* @param $crawlered
...
@@ -301,7 +302,7 @@ class TrackRepository extends BaseRepository
...
@@ -301,7 +302,7 @@ class TrackRepository extends BaseRepository
*
*
* @return array
* @return array
*/
*/
public
function
getTrackBySource
(
$src
=
'zing'
,
$crawlered
,
$limit
=
Constants
::
CRAWLER_LISTEN_LIMIT
,
$is_update_composer
=
false
)
public
function
getTrackBySource
(
$src
=
'zing'
,
$crawlered
,
$limit
=
Constants
::
CRAWLER_LISTEN_LIMIT
,
$is_update_composer
=
false
,
$blacklist_listen
=
null
)
{
{
$query
=
Track
::
select
([
'id'
,
'id_zing'
,
'id_nct'
,
'id_nhacvn'
,
'id_keeng'
,
'title'
,
'link_nct'
]);
$query
=
Track
::
select
([
'id'
,
'id_zing'
,
'id_nct'
,
'id_nhacvn'
,
'id_keeng'
,
'title'
,
'link_nct'
]);
...
@@ -309,19 +310,29 @@ class TrackRepository extends BaseRepository
...
@@ -309,19 +310,29 @@ class TrackRepository extends BaseRepository
$query
->
whereNotIn
(
'id'
,
$crawlered
);
$query
->
whereNotIn
(
'id'
,
$crawlered
);
}
}
if
(
is_array
(
$blacklist_listen
)
&&
count
(
$blacklist_listen
)
>
0
)
{
$query
->
whereNotIn
(
'id'
,
$blacklist_listen
);
}
switch
(
$src
)
{
switch
(
$src
)
{
case
'zing'
:
case
'zing'
:
$query
->
whereNotNull
(
'id_zing'
)
$query
->
whereNotNull
(
'id_zing'
)
->
where
(
'id_zing'
,
'<>'
,
''
)
->
where
(
'id_zing'
,
'<>'
,
''
);
->
whereNotIn
(
'id'
,
[
1139
,
3835
,
3664
]);
if
(
empty
(
$blacklist_listen
))
{
// Set mặc định
$query
->
whereNotIn
(
'id'
,
[
1139
,
3835
,
3664
]);
}
break
;
break
;
case
'nct'
:
case
'nct'
:
$query
->
whereNotNull
(
'id_nct'
)
$query
->
whereNotNull
(
'id_nct'
)
->
where
(
'id_nct'
,
'<>'
,
''
)
->
where
(
'id_nct'
,
'<>'
,
''
);
->
whereNotIn
(
'id'
,
[
1198
,
1700
,
1954
,
3223
,
3334
,
3357
,
3529
,
3688
,
3918
,
3664
,
3910
]);
if
(
$is_update_composer
)
{
if
(
$is_update_composer
)
{
$query
->
whereNull
(
'composer_list'
);
$query
->
whereNull
(
'composer_list'
);
}
}
if
(
empty
(
$blacklist_listen
))
{
// Set mặc định
$query
->
whereNotIn
(
'id'
,
[
1198
,
1700
,
1954
,
3223
,
3334
,
3357
,
3529
,
3688
,
3918
,
3664
,
3910
]);
}
break
;
break
;
case
'nhacvn'
:
$query
->
whereNotNull
(
'id_nhacvn'
)
->
where
(
'id_nhacvn'
,
'<>'
,
''
);
break
;
case
'nhacvn'
:
$query
->
whereNotNull
(
'id_nhacvn'
)
->
where
(
'id_nhacvn'
,
'<>'
,
''
);
break
;
case
'keeng'
:
$query
->
whereNotNull
(
'id_keeng'
)
->
where
(
'id_keeng'
,
'<>'
,
''
);
break
;
case
'keeng'
:
$query
->
whereNotNull
(
'id_keeng'
)
->
where
(
'id_keeng'
,
'<>'
,
''
);
break
;
...
@@ -443,5 +454,18 @@ class TrackRepository extends BaseRepository
...
@@ -443,5 +454,18 @@ class TrackRepository extends BaseRepository
return
$tracks
;
return
$tracks
;
}
}
/**
* Lấy ds blacklist để bỏ lấy lượt nghe
*
* @param $src
*
* @return array
*/
public
function
getBacklistListen
(
$src
)
{
$query
=
CrawlerListenBlacklist
::
where
(
'src'
,
strtolower
(
$src
))
->
pluck
(
'track_id'
);
return
$query
->
toArray
();
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment