Skip to content

C
crawler.vmusicchart.vn
Commit f3fbe27b by Phạm Văn Đoan
Options

cập nhật crawler bài hát ở 4 trang

parent 9182cb24
Showing with 228 additions and 183 deletions
app/Console/Commands/Keeng/KeengCrawlerTrackCommand.php 0 → 100644
<?php
namespace App\Console\Commands\Keeng;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class KeengCrawlerTrackCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'keeng:get-track';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lấy thông tin bài hát từ Keeng';
protected $trackRepository;
/**
* Create a new command instance.
*
* KeengCrawlerTrackCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
$this->trackRepository = $trackRepository;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
echo "\nBắt đầu xử lý lấy top 100 bài hát từ Keeng.";
// Lấy mảng ID của Zing những bài hát đã insert
$inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
// Lấy ds BXH về
$domain = config('api.keeng.domain');
$url = config('api.keeng.api_get_rank');
$curl = cURL($url);
$data = json_decode($curl);
$tracks = $data->data;
//$tracks = cURL($url);
//print_r($tracks);
//return false;
$bulk_track_insert = [];
$bulk_track_update = [];
$key_insert = 0;
$key_update = 0;
foreach ($tracks as $key => $track) {
if (isset($track->name)) {
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->name, $inserted)) {
$track_db = Track::where('title', $track->name)->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = isset($track->url) ? $track->url : null;
$track_db[0]->id_keeng = isset($track->id) ? $track->id : null;
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = isset($track->singer) ? $track->singer : null;
}
if (empty($track_db[0]->composer_list)) {
$track_db[0]->composer_list = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
$bulk_track_insert[$key_insert]['title'] = isset($track->name) ? $track->name : 'unknown-title';
$bulk_track_insert[$key_insert]['slug'] = isset($track->slug) ? $track->slug : null;
$bulk_track_insert[$key_insert]['userid'] = 0;
$bulk_track_insert[$key_insert]['tag'] = 'vmusic';
$bulk_track_insert[$key_insert]['genre'] = 1;
$bulk_track_insert[$key_insert]['download_hash'] = md5($track->id);
$bulk_track_insert[$key_insert]['time'] = time();
$bulk_track_insert[$key_insert]['link_keeng'] = isset($track->url) ? $track->url : null;
$bulk_track_insert[$key_insert]['id_keeng'] = isset($track->id) ? $track->id : null;
$bulk_track_insert[$key_insert]['singer_list'] = isset($track->singer) ? $track->singer : null;
$bulk_track_insert[$key_insert]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$bulk_track_insert[$key_insert]['src_thumbnail'] = isset($track->image) ? $track->image : null;
$bulk_track_insert[$key_insert]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
$bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
$key_insert++;
}
}
}
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
echo "\nKhông lấy được bài hát nào.";
}
// Ghi log
Crontjob::create(['message' => 'KeengCrawlerTrackCommand is called at ' . Carbon::now()]);
}
}
app/Console/Commands/Nct/NctCrawlerTrackCommand.php
...@@ -76,7 +76,9 @@ class NctCrawlerTrackCommand extends Command ...@@ -76,7 +76,9 @@ class NctCrawlerTrackCommand extends Command
// Cập nhật // Cập nhật
$track_db = Track::where('title', $track['title'])->take(1)->get(); $track_db = Track::where('title', $track['title'])->take(1)->get();
if (isset($track_db[0])) { if (isset($track_db[0])) {
$track_db[0]->track_duration = $duration; if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = $duration;
}
$track_db[0]->id_nct = $track['id_nct']; $track_db[0]->id_nct = $track['id_nct'];
$track_db[0]->id_nct_str = $track['id_nct_str']; $track_db[0]->id_nct_str = $track['id_nct_str'];
$track_db[0]->link_nct = $link; $track_db[0]->link_nct = $link;
......
app/Console/Commands/NhacVn/NhacVnCrawlerTrackCommand.php
...@@ -72,7 +72,9 @@ class NhacVnCrawlerTrackCommand extends Command ...@@ -72,7 +72,9 @@ class NhacVnCrawlerTrackCommand extends Command
if (isset($track_db[0])) { if (isset($track_db[0])) {
$track_db[0]->id_nhacvn = $track['id_nhacvn']; $track_db[0]->id_nhacvn = $track['id_nhacvn'];
$track_db[0]->link_nhacvn = $track['link']; $track_db[0]->link_nhacvn = $track['link'];
$track_db[0]->singer_list = $track['singer']; if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = $track['singer'];
}
$track_db[0]->crawler_at = Carbon::now(); $track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save(); $track_db[0]->save();
} }
......
app/Console/Commands/Zing/ZingCrawlerNewRealeaseCommand.php
...@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing; ...@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing;
use App\Helpers\Constants; use App\Helpers\Constants;
use App\Models\Crontjob; use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository; use App\Repositories\TrackRepository;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Console\Command; use Illuminate\Console\Command;
...@@ -69,25 +70,37 @@ class ZingCrawlerNewRealeaseCommand extends Command ...@@ -69,25 +70,37 @@ class ZingCrawlerNewRealeaseCommand extends Command
$artists = isset($track->artists) ? $track->artists : []; $artists = isset($track->artists) ? $track->artists : [];
if (is_array($artists) && count($artists) > 0) { if (is_array($artists) && count($artists) > 0) {
$artist_list = collect($artists)->map(function ($item){ return $item->name; })->all(); $artist_list = collect($artists)->map(function ($item){ return $item->name; })->all();
$singer_list = implode(',', $artist_list); $singer_list = implode('|', $artist_list);
} }
// Xử lý insert/update dựa vào tên bài hát // Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) { if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) {
$bulk_track_update[$key_update]['title'] = isset($track->title) ? $track->title : 'unknown-title'; $track_db = Track::where('title', $track->title)->take(1)->get();
$bulk_track_update[$key_update]['slug'] = isset($track->alias) ? $track->alias : null; if (isset($track_db[0])) {
$bulk_track_update[$key_update]['userid'] = 0; if (empty($track_db[0]->slug)) {
$bulk_track_update[$key_update]['tag'] = 'vmusic'; $track_db[0]->slug = isset($track->alias) ? $track->alias : null;
$bulk_track_update[$key_update]['genre'] = 1; }
$bulk_track_update[$key_update]['download_hash'] = md5($track->alias); if (empty($track_db[0]->download_hash)) {
$bulk_track_update[$key_update]['time'] = time(); $track_db[0]->download_hash = md5($track->alias);
$bulk_track_update[$key_update]['link'] = isset($track->link) ? $domain . $track->link : null; }
$bulk_track_update[$key_update]['id_zing'] = isset($track->id) ? $track->id : null; if (empty($track_db[0]->time)) {
$bulk_track_update[$key_update]['singer_list'] = $singer_list; $track_db[0]->time = time();
$bulk_track_update[$key_update]['track_duration'] = isset($track->duration) ? $track->duration : 0; }
$bulk_track_update[$key_update]['src_thumbnail'] = isset($track->thumbnail) ? $track->thumbnail : null; if (empty($track_db[0]->link)) {
$bulk_track_update[$key_update]['src_thumbnail_medium'] = isset($track->thumbnail_medium) ? $track->thumbnail_medium : null; $track_db[0]->link = isset($track->link) ? $domain . $track->link : null;
$bulk_track_update[$key_update]['link_mv'] = isset($track->mv_link) ? $domain . $track->mv_link : null; }
if (empty($track_db[0]->id_zing)) {
$track_db[0]->id_zing = isset($track->id) ? $track->id : null;
}
if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = isset($track->duration) ? $track->duration : 0;
}
if (empty($track_db[0]->link_mv)) {
$track_db[0]->link_mv = isset($track->mv_link) ? $domain . $track->mv_link : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++; $key_update++;
} else { } else {
...@@ -112,10 +125,10 @@ class ZingCrawlerNewRealeaseCommand extends Command ...@@ -112,10 +125,10 @@ class ZingCrawlerNewRealeaseCommand extends Command
} }
} }
echo "\nSố bài hát insert: " . count($bulk_track_insert); echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . count($bulk_track_update); echo "\nSố bài hát update: " . $key_update;
if (count($bulk_track_insert) > 0) { if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert); DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công."; echo "\nInsert thành công.";
} else { } else {
......
app/Console/Commands/Zing/ZingCrawlerTrackCommand.php
...@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing; ...@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing;
use App\Helpers\Constants; use App\Helpers\Constants;
use App\Models\Crontjob; use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository; use App\Repositories\TrackRepository;
use Carbon\Carbon; use Carbon\Carbon;
use Illuminate\Console\Command; use Illuminate\Console\Command;
...@@ -69,25 +70,37 @@ class ZingCrawlerTrackCommand extends Command ...@@ -69,25 +70,37 @@ class ZingCrawlerTrackCommand extends Command
$artists = isset($track->artists) ? $track->artists : []; $artists = isset($track->artists) ? $track->artists : [];
if (is_array($artists) && count($artists) > 0) { if (is_array($artists) && count($artists) > 0) {
$artist_list = collect($artists)->map(function ($item){ return $item->name; })->all(); $artist_list = collect($artists)->map(function ($item){ return $item->name; })->all();
$singer_list = implode(',', $artist_list); $singer_list = implode('|', $artist_list);
} }
// Xử lý insert/update dựa vào tên bài hát // Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) { if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) {
$bulk_track_update[$key_update]['title'] = isset($track->title) ? $track->title : 'unknown-title'; $track_db = Track::where('title', $track->title)->take(1)->get();
$bulk_track_update[$key_update]['slug'] = isset($track->alias) ? $track->alias : null; if (isset($track_db[0])) {
$bulk_track_update[$key_update]['userid'] = 0; if (empty($track_db[0]->slug)) {
$bulk_track_update[$key_update]['tag'] = 'vmusic'; $track_db[0]->slug = isset($track->alias) ? $track->alias : null;
$bulk_track_update[$key_update]['genre'] = 1; }
$bulk_track_update[$key_update]['download_hash'] = md5($track->alias); if (empty($track_db[0]->download_hash)) {
$bulk_track_update[$key_update]['time'] = time(); $track_db[0]->download_hash = md5($track->alias);
$bulk_track_update[$key_update]['link'] = isset($track->link) ? $domain . $track->link : null; }
$bulk_track_update[$key_update]['id_zing'] = isset($track->id) ? $track->id : null; if (empty($track_db[0]->time)) {
$bulk_track_update[$key_update]['singer_list'] = $singer_list; $track_db[0]->time = time();
$bulk_track_update[$key_update]['track_duration'] = isset($track->duration) ? $track->duration : 0; }
$bulk_track_update[$key_update]['src_thumbnail'] = isset($track->thumbnail) ? $track->thumbnail : null; if (empty($track_db[0]->link)) {
$bulk_track_update[$key_update]['src_thumbnail_medium'] = isset($track->thumbnail_medium) ? $track->thumbnail_medium : null; $track_db[0]->link = isset($track->link) ? $domain . $track->link : null;
$bulk_track_update[$key_update]['link_mv'] = isset($track->mv_link) ? $domain . $track->mv_link : null; }
if (empty($track_db[0]->id_zing)) {
$track_db[0]->id_zing = isset($track->id) ? $track->id : null;
}
if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = isset($track->duration) ? $track->duration : 0;
}
if (empty($track_db[0]->link_mv)) {
$track_db[0]->link_mv = isset($track->mv_link) ? $domain . $track->mv_link : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++; $key_update++;
} else { } else {
...@@ -112,10 +125,10 @@ class ZingCrawlerTrackCommand extends Command ...@@ -112,10 +125,10 @@ class ZingCrawlerTrackCommand extends Command
} }
} }
echo "\nSố bài hát insert: " . count($bulk_track_insert); echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . count($bulk_track_update); echo "\nSố bài hát update: " . $key_update;
if (count($bulk_track_insert) > 0) { if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert); DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công."; echo "\nInsert thành công.";
} else { } else {
......
app/Console/Kernel.php
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
namespace App\Console; namespace App\Console;
use App\Console\Commands\CrontjobCommand; use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
use App\Console\Commands\Nct\NctCrawlerTrackCommand; use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand; use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand; use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand;
...@@ -22,10 +23,10 @@ class Kernel extends ConsoleKernel ...@@ -22,10 +23,10 @@ class Kernel extends ConsoleKernel
CrontjobCommand::class, CrontjobCommand::class,
ZingCrawlerTrackCommand::class, ZingCrawlerTrackCommand::class,
ZingCrawlerNewRealeaseCommand::class, ZingCrawlerNewRealeaseCommand::class,
CrawlerTrackImageCommand::class,
NctCrawlerTrackCommand::class, NctCrawlerTrackCommand::class,
NhacVnCrawlerTrackCommand::class, NhacVnCrawlerTrackCommand::class,
KeengCrawlerTrackCommand::class,
CrawlerTrackImageCommand::class,
]; ];
...@@ -37,19 +38,18 @@ class Kernel extends ConsoleKernel ...@@ -37,19 +38,18 @@ class Kernel extends ConsoleKernel
*/ */
protected function schedule(Schedule $schedule) protected function schedule(Schedule $schedule)
{ {
// $schedule->command('inspire')
// ->hourly();
$schedule->command('crontjob:test')->hourly(); $schedule->command('crontjob:test')->hourly();
$schedule->command('zing:get-track')->everyThirtyMinutes(); $schedule->command('zing:get-track')->everyThirtyMinutes();
$schedule->command('zing:get-new-release-track')->hourlyAt(5); $schedule->command('zing:get-new-release-track')->hourlyAt(5);
$schedule->command('crawler:get-track-image')->hourlyAt(10); $schedule->command('nct:get-track')->hourlyAt(10);
$schedule->command('nhacvn:get-track')->hourlyAt(15);
$schedule->command('nct:get-track')->hourlyAt(25); $schedule->command('keeng:get-track')->hourlyAt(20);
$schedule->command('nhacvn:get-track')->hourlyAt(35); $schedule->command('crawler:get-track-image')->hourlyAt(35);
$schedule->command('queue:work --once --timeout=120')->everyMinute()->withoutOverlapping(); $schedule->command('queue:work --once --timeout=120')->everyMinute()->withoutOverlapping();
} }
......
app/Helpers/Functions.php
...@@ -720,148 +720,32 @@ if (!function_exists('getUrlArtistByPage')) { ...@@ -720,148 +720,32 @@ if (!function_exists('getUrlArtistByPage')) {
} }
} }
if (!function_exists('read_money')) { if (!function_exists('cURL')) {
/** function cURL($url) {
* Hàm đọc số tiền trên hóa đơn thành chuỗi $ch = curl_init();
* doanpv - 14/06/2019
*
* @param $money
* @param string $currency_type
*
* @return string
*/
function read_money($money, $currency_type='1')
{
switch (strtoupper($currency_type)) {
case '1': // VND
$unit_name = 'đồng';
$sub_unit_name = 'xu';
$money = round($money, 0);
break;
case '2': // USD
$unit_name = 'Đô la Mỹ';
$sub_unit_name = 'cent';
$money = round($money, 2);
break;
case '3': // EUR
$unit_name = 'Ơ Rô';
$sub_unit_name = 'cent';
$money = round($money, 2);
break;
default:
$unit_name = 'đồng';
$sub_unit_name = 'xu';
$money = round($money, 0);
}
if ($money == 0) {
return 'Không ' . $unit_name . '.';
} elseif ($money < 0) {
return './.';
}
$number = [
1 => 'một',
2 => 'hai',
3 => 'ba',
4 => 'bốn',
5 => 'năm',
6 => 'sáu',
7 => 'bảy',
8 => 'tám',
9 => 'chín',
];
if ($money <= 2147483647) {
$odd = intval(($money - intval($money))*100);
$even = intval(trim($money));
$even = str_repeat(' ', 15 - strlen($even)) . $even;
$thousands_of_billions = intval(substr($even, 0, 3));
$billions = intval(substr($even, 3, 3));
$millions = intval(substr($even, 6, 3));
$thousands = intval(substr($even, 9, 3));
$units = intval(substr($even, 12, 3));
} else {
$odd = floatval(($money - floatval($money))*100);
$even = floatval(trim($money));
$even = str_repeat(' ', 15 - strlen($even)) . $even;
$thousands_of_billions = floatval(substr($even, 0, 3));
$billions = floatval(substr($even, 3, 3));
$millions = floatval(substr($even, 6, 3));
$thousands = floatval(substr($even, 9, 3));
$units = floatval(substr($even, 12, 3));
}
if ($thousands_of_billions == 0 && $billions == 0 && $millions == 0 && $thousands == 0 && $units == 0) {
$result = 'Không ' . $unit_name . ' ';
$i = 5;
} else {
$result = '';
$i = 0;
}
while ($i <= 5) {
switch ($i) {
case 0:
$temp = $thousands_of_billions;
$name = 'nghìn tỷ'; break;
case 1:
$temp = $billions;
$name = 'tỷ'; break;
case 2:
$temp = $millions;
$name = 'triệu'; break;
case 3:
$temp = $thousands;
$name = 'nghìn'; break;
case 4:
$temp = $units;
$name = $unit_name; break;
case 5:
$temp = $odd;
$name = $sub_unit_name; break;
}
if ($temp != 0) {
$hundred = intval($temp/100);
$ten = intval(($temp - $hundred*100)/10);
$unit = intval(($temp - $hundred * 100) - $ten*10);
if (substr($result, strlen($result) - 3) == ' ') {
$result = substr($result, strlen($result) - 3);
}
$result .= ((strlen($result) == 0) ? '' : ', ') . (($hundred != 0) ? $number[$hundred] . ' trăm ' : '');
if ($ten == 0 && $hundred != 0 && $unit != 0) { curl_setopt($ch, CURLOPT_URL, $url);
$result .= 'lẻ '; curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
} else { curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
if($ten != 0) {
$result .= (($ten != 0 && $ten != 1) ? $number[$ten] . ' mươi ' : 'mười ');
}
}
if ($ten != 0 && $unit == 5) { curl_setopt($ch, CURLOPT_ENCODING, 'gzip, deflate');
$result .= 'lăm ' . $name . ' ';
} else {
if ($ten > 1 && $unit == 1) {
$result .= 'mốt ' . $name . ' ';
} else {
$result .= (($unit != 0) ? $number[$unit] . ' ' . $name : $name) . ' ';
}
}
} else {
$result .= (($i == 4) ? $unit_name . '': '');
}
$i = $i + 1; //$headers = array();
} //$headers[] = 'Authority: mp3.zing.vn';
//$headers[] = 'Cache-Control: max-age=0';
//$headers[] = 'Upgrade-Insecure-Requests: 1';
//$headers[] = 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36';
//$headers[] = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3';
//$headers[] = 'Accept-Encoding: gzip, deflate, br';
//$headers[] = 'Accept-Language: vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7,ja;q=0.6';
//curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
if ($odd == 0 && $currency_type == '1') { $result = curl_exec($ch);
$result .= ((substr($result, strlen($result) - 3) == ' ') ? '' : ' ') . 'chẵn'; if (curl_errno($ch)) {
return false;
} }
curl_close ($ch);
return ucfirst(str_replace(' ,', ',', $result)) . './.'; return $result;
} }
} }
......
config/api.php
...@@ -79,6 +79,12 @@ return [ ...@@ -79,6 +79,12 @@ return [
'ranking20' => 'https://nhac.vn/bang-xep-hang-bai-hat-viet-nam-bxdE' 'ranking20' => 'https://nhac.vn/bang-xep-hang-bai-hat-viet-nam-bxdE'
], ],
'keeng' => [
'domain' => 'http://keeng.vn/',
'ranking20' => 'http://keeng.vn/bang-xep-hang/song/viet-nam',
'api_get_rank' => 'http://vip.service.keeng.vn:8080/KeengWSRestful//ws/common/getRankDetail?item_type=1&rank_type=50'
],
]; ];
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment