Skip to content

C
crawler.vmusicchart.vn
Commit 4131a4ba by Phạm Văn Đoan
Options

dọn code, thiết lập timeout

parent f63f3efd
Showing with 354 additions and 13 deletions
app/Console/Commands/CrawlerTrackImageCommand.php
......@@ -36,6 +36,7 @@ class CrawlerTrackImageCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......@@ -46,7 +47,7 @@ class CrawlerTrackImageCommand extends Command
*/
public function handle()
{
$tracks = $this->trackRepository->getTrackImageEmpty();
$tracks = $this->trackRepository->getTrackImageEmpty(200);
echo "\nTổng số bài hát cần lấy ảnh về: " . count($tracks);
......
app/Console/Commands/CrawlerTrackListenCommand.php
......@@ -35,6 +35,7 @@ class CrawlerTrackListenCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......@@ -90,6 +91,12 @@ class CrawlerTrackListenCommand extends Command
['today_nct' => $listen, 'nct_crawler_at' => Carbon::now()]
);
break;
case 'keeng':
CrawlerListen::updateOrCreate(
['track_id' => $track_id],
['today_keeng' => $listen, 'keeng_crawler_at' => Carbon::now()]
);
break;
}
}
}
......@@ -134,6 +141,18 @@ class CrawlerTrackListenCommand extends Command
}
break;
case 'keeng':
$url = getKeengUrlGetCounter($track['id_keeng']);
$curl = cURL($url);
$response = json_decode($curl);
if (isset($response->data)) {
if (isset($response->data->listen_no) && $response->data->listen_no > 0) {
$listen = $response->data->listen_no;
}
}
break;
}
return $listen;
......
app/Console/Commands/Keeng/KeengCrawlerByCateCommand.php 0 → 100644
<?php
namespace App\Console\Commands\Keeng;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class KeengCrawlerByCateCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'keeng:get-track-by-cate';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lấy những bài hát mới phát hành';
protected $trackRepository;
/**
* Create a new command instance.
*
* KeengCrawlerByCateCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
echo "\nBắt đầu xử lý lấy 100 bài hát theo từng trang từ Keeng.";
// Lấy mảng ID của Zing những bài hát đã insert
$inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
// Lấy ds BXH về
$domain = config('api.keeng.domain');
$tracks = [];
$index = 0;
for ($i = 1; $i <= 10; $i++) {
$url = getKeengUrlCategoryByPopViet($i);
$curl = cURL($url);
$data = json_decode($curl);
if (isset($data->data)) {
foreach ($data->data as $key => $track) {
$tracks[$index]['title'] = isset($track->name) ? $track->name : 'unknown-title';
$tracks[$index]['slug'] = isset($track->slug) ? $track->slug : null;
$tracks[$index]['userid'] = 0;
$tracks[$index]['tag'] = 'vmusic';
$tracks[$index]['genre'] = 1;
$tracks[$index]['download_hash'] = md5($track->id);
$tracks[$index]['time'] = time();
$tracks[$index]['link_keeng'] = isset($track->url) ? $track->url : null;
$tracks[$index]['id_keeng'] = isset($track->identify) ? $track->identify : null;
$tracks[$index]['singer_list'] = isset($track->singer) ? $track->singer : null;
$tracks[$index]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$tracks[$index]['src_thumbnail'] = isset($track->image) ? $track->image : null;
$tracks[$index]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
$index++;
}
}
}
$bulk_track_insert = [];
$bulk_track_update = [];
$key_insert = 0;
$key_update = 0;
foreach ($tracks as $key => $track) {
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track['title'], $inserted)) {
$track_db = Track::where('title', $track['title'])->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = $track['link_keeng'];
$track_db[0]->id_keeng = $track['id_keeng'];
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = $track['singer_list'];
}
if (empty($track_db[0]->composer_list)) {
$track_db[0]->composer_list = $track['composer_list'];
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
$bulk_track_insert[$key_insert]['title'] = $track['title'];
$bulk_track_insert[$key_insert]['slug'] = $track['slug'];
$bulk_track_insert[$key_insert]['userid'] = 0;
$bulk_track_insert[$key_insert]['tag'] = 'vmusic';
$bulk_track_insert[$key_insert]['genre'] = 1;
$bulk_track_insert[$key_insert]['download_hash'] = md5($track['download_hash']);
$bulk_track_insert[$key_insert]['time'] = time();
$bulk_track_insert[$key_insert]['link_keeng'] = $track['link_keeng'];
$bulk_track_insert[$key_insert]['id_keeng'] = $track['id_keeng'];
$bulk_track_insert[$key_insert]['singer_list'] = $track['singer_list'];
$bulk_track_insert[$key_insert]['composer_list'] = $track['composer_list'];
$bulk_track_insert[$key_insert]['src_thumbnail'] = $track['src_thumbnail'];
$bulk_track_insert[$key_insert]['src_thumbnail_medium'] = $track['src_thumbnail_medium'];
$bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
$key_insert++;
}
}
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
echo "\nKhông lấy được bài hát nào.";
}
// Ghi log
Crontjob::create(['message' => 'KeengCrawlerByCateCommand is called at ' . Carbon::now()]);
}
}
app/Console/Commands/Keeng/KeengCrawlerNewRealeaseCommand.php 0 → 100644
<?php
namespace App\Console\Commands\Keeng;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class KeengCrawlerNewRealeaseCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'keeng:get-new-release-track';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lấy những bài hát mới phát hành';
protected $trackRepository;
/**
* Create a new command instance.
*
* KeengCrawlerNewRealeaseCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
echo "\nBắt đầu xử lý lấy 100 bài hát theo từng trang từ Keeng.";
// Lấy mảng ID của Zing những bài hát đã insert
$inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
// Lấy ds BXH về
$domain = config('api.keeng.domain');
$tracks = [];
$index = 0;
for ($i = 1; $i <= 10; $i++) {
$url = getKeengUrlNewRealease($i);
$curl = cURL($url);
$data = json_decode($curl);
if (isset($data->data)) {
foreach ($data->data as $key => $track) {
$tracks[$index]['title'] = isset($track->name) ? $track->name : 'unknown-title';
$tracks[$index]['slug'] = isset($track->slug) ? $track->slug : null;
$tracks[$index]['userid'] = 0;
$tracks[$index]['tag'] = 'vmusic';
$tracks[$index]['genre'] = 1;
$tracks[$index]['download_hash'] = md5($track->id);
$tracks[$index]['time'] = time();
$tracks[$index]['link_keeng'] = isset($track->url) ? $track->url : null;
$tracks[$index]['id_keeng'] = isset($track->identify) ? $track->identify : null;
$tracks[$index]['singer_list'] = isset($track->singer) ? $track->singer : null;
$tracks[$index]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$tracks[$index]['src_thumbnail'] = isset($track->image) ? $track->image : null;
$tracks[$index]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
$index++;
}
}
}
$bulk_track_insert = [];
$bulk_track_update = [];
$key_insert = 0;
$key_update = 0;
foreach ($tracks as $key => $track) {
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track['title'], $inserted)) {
$track_db = Track::where('title', $track['title'])->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = $track['link_keeng'];
$track_db[0]->id_keeng = $track['id_keeng'];
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = $track['singer_list'];
}
if (empty($track_db[0]->composer_list)) {
$track_db[0]->composer_list = $track['composer_list'];
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
$bulk_track_insert[$key_insert]['title'] = $track['title'];
$bulk_track_insert[$key_insert]['slug'] = $track['slug'];
$bulk_track_insert[$key_insert]['userid'] = 0;
$bulk_track_insert[$key_insert]['tag'] = 'vmusic';
$bulk_track_insert[$key_insert]['genre'] = 1;
$bulk_track_insert[$key_insert]['download_hash'] = md5($track['download_hash']);
$bulk_track_insert[$key_insert]['time'] = time();
$bulk_track_insert[$key_insert]['link_keeng'] = $track['link_keeng'];
$bulk_track_insert[$key_insert]['id_keeng'] = $track['id_keeng'];
$bulk_track_insert[$key_insert]['singer_list'] = $track['singer_list'];
$bulk_track_insert[$key_insert]['composer_list'] = $track['composer_list'];
$bulk_track_insert[$key_insert]['src_thumbnail'] = $track['src_thumbnail'];
$bulk_track_insert[$key_insert]['src_thumbnail_medium'] = $track['src_thumbnail_medium'];
$bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
$key_insert++;
}
}
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
echo "\nKhông lấy được bài hát nào.";
}
// Ghi log
Crontjob::create(['message' => 'KeengCrawlerNewRealeaseCommand is called at ' . Carbon::now()]);
}
}
app/Console/Commands/Keeng/KeengCrawlerTrackCommand.php
......@@ -37,6 +37,7 @@ class KeengCrawlerTrackCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......@@ -47,7 +48,7 @@ class KeengCrawlerTrackCommand extends Command
*/
public function handle()
{
echo "\nBắt đầu xử lý lấy top 100 bài hát từ Keeng.";
echo "\nBắt đầu xử lý lấy top bài hát từ Keeng.";
// Lấy mảng ID của Zing những bài hát đã insert
$inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
......@@ -59,12 +60,6 @@ class KeengCrawlerTrackCommand extends Command
$data = json_decode($curl);
$tracks = $data->data;
//$tracks = cURL($url);
//print_r($tracks);
//return false;
$bulk_track_insert = [];
$bulk_track_update = [];
$key_insert = 0;
......@@ -76,7 +71,7 @@ class KeengCrawlerTrackCommand extends Command
$track_db = Track::where('title', $track->name)->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = isset($track->url) ? $track->url : null;
$track_db[0]->id_keeng = isset($track->id) ? $track->id : null;
$track_db[0]->id_keeng = isset($track->identify) ? $track->identify : null;
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = isset($track->singer) ? $track->singer : null;
}
......@@ -97,7 +92,7 @@ class KeengCrawlerTrackCommand extends Command
$bulk_track_insert[$key_insert]['download_hash'] = md5($track->id);
$bulk_track_insert[$key_insert]['time'] = time();
$bulk_track_insert[$key_insert]['link_keeng'] = isset($track->url) ? $track->url : null;
$bulk_track_insert[$key_insert]['id_keeng'] = isset($track->id) ? $track->id : null;
$bulk_track_insert[$key_insert]['id_keeng'] = isset($track->identify) ? $track->identify : null;
$bulk_track_insert[$key_insert]['singer_list'] = isset($track->singer) ? $track->singer : null;
$bulk_track_insert[$key_insert]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$bulk_track_insert[$key_insert]['src_thumbnail'] = isset($track->image) ? $track->image : null;
......
app/Console/Commands/Nct/NctCrawlerTrackCommand.php
......@@ -37,6 +37,7 @@ class NctCrawlerTrackCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......
app/Console/Commands/NhacVn/NhacVnCrawlerTrackCommand.php
......@@ -37,6 +37,7 @@ class NhacVnCrawlerTrackCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......
app/Console/Commands/Zing/ZingCrawlerNewRealeaseCommand.php
......@@ -37,6 +37,7 @@ class ZingCrawlerNewRealeaseCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......
app/Console/Commands/Zing/ZingCrawlerTrackCommand.php
......@@ -37,6 +37,7 @@ class ZingCrawlerTrackCommand extends Command
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
ini_set('max_execution_time', -1);
$this->trackRepository = $trackRepository;
}
......
app/Console/Kernel.php
......@@ -4,6 +4,8 @@ namespace App\Console;
use App\Console\Commands\CrawlerTrackListenCommand;
use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\Keeng\KeengCrawlerByCateCommand;
use App\Console\Commands\Keeng\KeengCrawlerNewRealeaseCommand;
use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
......@@ -31,7 +33,9 @@ class Kernel extends ConsoleKernel
CrawlerTrackImageCommand::class,
//ZingCrawlerListenCommand::class,
CrawlerTrackListenCommand::class
CrawlerTrackListenCommand::class,
KeengCrawlerByCateCommand::class,
KeengCrawlerNewRealeaseCommand::class,
];
......
app/Helpers/Functions.php
......@@ -227,3 +227,43 @@ if (!function_exists('getKeengUrlGetCounter')) {
return $url;
}
}
if (!function_exists('getKeengUrlCategoryByPopViet')) {
/**
* Hàm tạo URL để lấy thông tin bài hát theo chủ đề Pop Việt
* http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getCategoryDetail?page=100&num=100&slug=pop-viet&type=1
*
* @param $page
* @param $num
* @param $slug
* @param $type
* @return string
*/
function getKeengUrlCategoryByPopViet($page = 1, $num = 100, $slug = 'pop-viet', $type = 1) {
$url = 'http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getCategoryDetail?';
$url .= 'page=' . $page;
$url .= '&num=' . $num;
$url .= '&slug=' . $slug;
$url .= '&type=' . $type;
return $url;
}
}
if (!function_exists('getKeengUrlNewRealease')) {
/**
* Hàm tạo URL để lấy thông tin bài hát theo chủ đề Pop Việt
* http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getListSongNewV4?page=1&num=40
*
* @param $page
* @param $num
* @return string
*/
function getKeengUrlNewRealease($page = 1, $num = 100) {
$url = 'http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getListSongNewV4?';
$url .= 'page=' . $page;
$url .= '&num=' . $num;
return $url;
}
}
app/Repositories/TrackRepository.php
......@@ -59,11 +59,11 @@ class TrackRepository extends BaseRepository
*
* @return array
*/
public function getTrackImageEmpty()
public function getTrackImageEmpty($limit=200)
{
$query = Track::select(['id', 'art', 'id_zing', 'id_nct', 'id_keeng', 'src_thumbnail_medium'])
->whereNull('art')
->whereNotNull('src_thumbnail_medium');
->whereNotNull('src_thumbnail_medium')->take($limit);
return $query->get()->toArray();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment