Skip to content

C
crawler.vmusicchart.vn
Commit 8c349671 by Phạm Văn Đoan
Options

cập nhật lấy lượt view ở các trang zing, nct, keeng

parent 4131a4ba
Showing with 204 additions and 13 deletions
app/Console/Commands/CalculateDailyListenCommand.php 0 → 100644
<?php
namespace App\Console\Commands;
use App\Models\Crontjob;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
class CalculateDailyListenCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'calculate:daily-listen';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Di chuyển lượt nghe sang ngày hôm qua trước khi cập nhật lượt nghe ngày mới';
protected $trackRepository;
/**
* Create a new command instance.
*
* CalculateDailyListenCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
/**
* Xử lý di chuyển dựa vào cờ date_moved_at
* Mỗi lần di chuyển 1k bản ghi trong crawler_listens
* Di chuyển xong thì cập nhật ngày trong crawler_listens.date_moved_at và cho lượt nghe hôm nay=0
* Trong lần sau sẽ ko lấy những bản ghi có date_moved_at = ngày hiện tại
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
// Ghi log xem có gọi không
Crontjob::create(['message' => 'CalculateDailyListenCommand is called at ' . Carbon::now()]);
// Xử lý di chuyển
echo "\n Bắt đầu xử lý lấy lượt nghe ngày hôm qua";
$crawlered = $this->trackRepository->getCrawlerListenNotYetMoved();
if ($crawlered) {
echo "\n Số lượng cần tính toán lượt nghe: " . $crawlered->count();
foreach ($crawlered as $key => $listen) {
// Xử lý lượt nghe từ zing -----------------------------------------------------------------------------
if ($listen->yesterday_zing == 0) {
if ($listen->today_zing > 0) {
$listen->yesterday_zing = $listen->today_zing;
$listen->today_zing = 0;
}
} else if ($listen->yesterday_zing > 0) {
if ($listen->today_zing > 0) {
$difference = $listen->today_zing - $listen->yesterday_zing;
if ($difference >= 0) {
$listen->yesterday_zing = $listen->today_zing;
$listen->today_zing = 0;
$listen->date_zing = $difference;
$listen->increment('week_zing', $difference);
$listen->increment('month_zing', $difference);
$listen->increment('year_zing', $difference);
$listen->increment('alltime_zing', $difference);
}
}
}
// Xử lý lượt nghe từ NCT ------------------------------------------------------------------------------
if ($listen->yesterday_nct == 0) {
if ($listen->today_nct > 0) {
$listen->yesterday_nct = $listen->today_nct;
$listen->today_nct = 0;
}
} else if ($listen->yesterday_nct > 0) {
if ($listen->today_nct > 0) {
$difference = $listen->today_nct - $listen->yesterday_nct;
if ($difference >= 0) {
$listen->yesterday_nct = $listen->today_nct;
$listen->today_nct = 0;
$listen->date_nct = $difference;
$listen->increment('week_nct', $difference);
$listen->increment('month_nct', $difference);
$listen->increment('year_nct', $difference);
$listen->increment('alltime_nct', $difference);
}
}
}
// Xử lý lượt nghe từ nhacvn ---------------------------------------------------------------------------
if ($listen->yesterday_keeng == 0) {
if ($listen->today_keeng > 0) {
$listen->yesterday_keeng = $listen->today_keeng;
$listen->today_keeng = 0;
}
} else if ($listen->yesterday_keeng > 0) {
if ($listen->today_keeng > 0) {
$difference = $listen->today_keeng - $listen->yesterday_keeng;
if ($difference >= 0) {
$listen->yesterday_keeng = $listen->today_keeng;
$listen->today_keeng = 0;
$listen->date_keeng = $difference;
$listen->increment('week_keeng', $difference);
$listen->increment('month_keeng', $difference);
$listen->increment('year_keeng', $difference);
$listen->increment('alltime_keeng', $difference);
}
}
}
// Lưu lại giá trị -------------------------------------------------------------------------------------
$listen->date_moved_at = Carbon::now();
$listen->save();
echo "\n Đã tính toán lượt nghe bản ghi có ID= " . $listen->id . ' (Track ID: ' . $listen->track_id . ')';
}
}
echo "\n Tính toán kết thúc kết thúc: " . $crawlered->count();
}
}
app/Console/Commands/CrawlerTrackListenCommand.php
......@@ -2,6 +2,7 @@
namespace App\Console\Commands;
use App\Helpers\Constants;
use App\Models\CrawlerListen;
use App\Models\Crontjob;
use App\Repositories\TrackRepository;
......@@ -15,7 +16,7 @@ class CrawlerTrackListenCommand extends Command
*
* @var string
*/
protected $signature = 'crawler:get-listen {from_page?}';
protected $signature = 'crawler:get-listen {from_page?} {--limit=}';
/**
* The console command description.
......@@ -50,11 +51,18 @@ class CrawlerTrackListenCommand extends Command
Crontjob::create(['message' => 'CrawlerTrackListenCommand is called at ' . Carbon::now()]);
$from_page = $this->argument('from_page');
$limit = $this->option('limit');
if (empty($from_page) || !in_array($from_page, ['zing', 'nct', 'nhacvn', 'keeng'])) {
$from_page = $this->choice('Chọn trang nhạc ', ['zing', 'nct', 'nhacvn', 'keeng']);
}
if (empty($limit)) {
$limit = Constants::CRAWLER_LISTEN_LIMIT;
} else {
$limit = -1;
}
echo "\n Bắt đầu xử lý lấy lượt nghe từ: " . $from_page;
$crawlered = $this->trackRepository->getCrawlerListenInserted($from_page, false);
......@@ -62,18 +70,19 @@ class CrawlerTrackListenCommand extends Command
echo "\n Tổng số bài hát đã lấy lượt nghe hôm nay: " . count($crawlered);
// Lấy mảng bài hát theo từng hệ thống nhạc
$inserted = $this->trackRepository->getTrackBySource($from_page, $crawlered);
$inserted = $this->trackRepository->getTrackBySource($from_page, $crawlered, $limit);
$total = count($inserted);
echo "\n Tổng số bài hát cần lấy lượt nghe: " . count($inserted);
echo "\n Tổng số bài hát cần lấy lượt nghe: " . $total;
if (count($inserted) > 0) {
if ($total > 0) {
$data_listen = [];
foreach ($inserted as $key => $track) {
$listen = $this->privateGetListen($from_page, $track);
if ($listen > 0) {
$data_listen[$track['id']] = $listen;
}
echo "\n " . ($key + 1) . ". " . $track['title'] . " (Track ID= " . $track['id'] . "): " . $listen;
echo "\n " . ($key + 1) . "/" . $total . ". " . $track['title'] . " (Track ID= " . $track['id'] . "): " . $listen;
}
if (count($data_listen) > 0) {
......
app/Console/Kernel.php
......@@ -2,6 +2,7 @@
namespace App\Console;
use App\Console\Commands\CalculateDailyListenCommand;
use App\Console\Commands\CrawlerTrackListenCommand;
use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\Keeng\KeengCrawlerByCateCommand;
......@@ -9,7 +10,6 @@ use App\Console\Commands\Keeng\KeengCrawlerNewRealeaseCommand;
use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
use App\Console\Commands\Zing\ZingCrawlerListenCommand;
use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand;
use App\Console\Commands\Zing\ZingCrawlerTrackCommand;
use App\Console\Commands\CrawlerTrackImageCommand;
......@@ -25,18 +25,20 @@ class Kernel extends ConsoleKernel
*/
protected $commands = [
CrontjobCommand::class,
ZingCrawlerTrackCommand::class,
ZingCrawlerNewRealeaseCommand::class,
NctCrawlerTrackCommand::class,
NhacVnCrawlerTrackCommand::class,
KeengCrawlerTrackCommand::class,
CrawlerTrackImageCommand::class,
//ZingCrawlerListenCommand::class,
CrawlerTrackListenCommand::class,
KeengCrawlerByCateCommand::class,
KeengCrawlerNewRealeaseCommand::class,
CrawlerTrackImageCommand::class,
CrawlerTrackListenCommand::class,
CalculateDailyListenCommand::class,
];
......@@ -61,6 +63,27 @@ class Kernel extends ConsoleKernel
$schedule->command('crawler:get-track-image')->hourlyAt(35);
// 22h bắt đầu quét. Quét 3 lần, lần cuối quét hết
$schedule->command('crawler:get-listen keeng')->dailyAt('22:00');
$schedule->command('crawler:get-listen keeng')->dailyAt('22:05');
$schedule->command('crawler:get-listen keeng')->dailyAt('22:10');
$schedule->command('crawler:get-listen keeng --limit=-1')->dailyAt('22:15');
// 22h30 bắt đầu quét. Quét 3 lần, lần cuối quét hết
$schedule->command('crawler:get-listen nct')->dailyAt('22:30');
$schedule->command('crawler:get-listen nct')->dailyAt('22:35');
$schedule->command('crawler:get-listen nct')->dailyAt('22:40');
$schedule->command('crawler:get-listen nct --limit=-1')->dailyAt('22:45');
// 23h bắt đầu quét. Quét 3 lần, lần cuối quét hết
$schedule->command('crawler:get-listen zing')->dailyAt('23:00');
$schedule->command('crawler:get-listen zing')->dailyAt('23:05');
$schedule->command('crawler:get-listen zing')->dailyAt('23:10');
$schedule->command('crawler:get-listen zing --limit=-1')->dailyAt('23:15');
$schedule->command('calculate:daily-listen')->dailyAt('1:00');
$schedule->command('calculate:daily-listen')->dailyAt('1:10');
$schedule->command('queue:work --once --timeout=120')->everyMinute()->withoutOverlapping();
}
......
app/Models/CrawlerListen.php
......@@ -20,7 +20,8 @@ class CrawlerListen extends Model
'month_zing','month_nct','week_nhacvn','month_keeng',
'year_zing','year_nct','year_nhacvn','year_keeng',
'alltime_zing','alltime_nct','alltime_nhacvn','alltime_keeng',
'zing_crawler_at','nct_crawler_at','nhacvn_crawler_at','keeng_crawler_at'
'zing_crawler_at','nct_crawler_at','nhacvn_crawler_at','keeng_crawler_at',
'date_moved_at'
];
}
app/Repositories/TrackRepository.php
......@@ -63,7 +63,8 @@ class TrackRepository extends BaseRepository
{
$query = Track::select(['id', 'art', 'id_zing', 'id_nct', 'id_keeng', 'src_thumbnail_medium'])
->whereNull('art')
->whereNotNull('src_thumbnail_medium')->take($limit);
->whereNotNull('src_thumbnail_medium')
->take($limit);
return $query->get()->toArray();
}
......@@ -307,7 +308,9 @@ class TrackRepository extends BaseRepository
case 'keeng': $query->whereNotNull('id_keeng')->where('id_keeng', '<>', ''); break;
}
$query->take(($limit < 1) ? Constants::CRAWLER_LISTEN_LIMIT : $limit);
if ($limit > 0) {
$query->take($limit);
}
return $query->get()->toArray();
......@@ -364,5 +367,20 @@ class TrackRepository extends BaseRepository
return collect($result)->map(function ($item) { return $item->track_id; })->all();
}
public function getCrawlerListenNotYetMoved($max_date=null)
{
if (empty($max_date)) {
$max_date = date('Y-m-d 00:00:00');
} else {
$max_date = date('Y-m-d 23:59:59', strtotime($max_date));
}
return CrawlerListen::select('*')
->where('date_moved_at', '<=', $max_date)
->orWhereNull('date_moved_at')
->take(5000)
->get();
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment