Skip to content

C
crawler.vmusicchart.vn
Commit f3fbe27b by Phạm Văn Đoan
Options

cập nhật crawler bài hát ở 4 trang

parent 9182cb24
Showing with 228 additions and 183 deletions
app/Console/Commands/Keeng/KeengCrawlerTrackCommand.php 0 → 100644
<?php
namespace App\Console\Commands\Keeng;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
class KeengCrawlerTrackCommand extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'keeng:get-track';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Lấy thông tin bài hát từ Keeng';
protected $trackRepository;
/**
* Create a new command instance.
*
* KeengCrawlerTrackCommand constructor.
* @param TrackRepository $trackRepository
*/
public function __construct(TrackRepository $trackRepository)
{
parent::__construct();
$this->trackRepository = $trackRepository;
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
echo "\nBắt đầu xử lý lấy top 100 bài hát từ Keeng.";
// Lấy mảng ID của Zing những bài hát đã insert
$inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
// Lấy ds BXH về
$domain = config('api.keeng.domain');
$url = config('api.keeng.api_get_rank');
$curl = cURL($url);
$data = json_decode($curl);
$tracks = $data->data;
//$tracks = cURL($url);
//print_r($tracks);
//return false;
$bulk_track_insert = [];
$bulk_track_update = [];
$key_insert = 0;
$key_update = 0;
foreach ($tracks as $key => $track) {
if (isset($track->name)) {
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->name, $inserted)) {
$track_db = Track::where('title', $track->name)->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->link_keeng = isset($track->url) ? $track->url : null;
$track_db[0]->id_keeng = isset($track->id) ? $track->id : null;
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = isset($track->singer) ? $track->singer : null;
}
if (empty($track_db[0]->composer_list)) {
$track_db[0]->composer_list = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
$bulk_track_insert[$key_insert]['title'] = isset($track->name) ? $track->name : 'unknown-title';
$bulk_track_insert[$key_insert]['slug'] = isset($track->slug) ? $track->slug : null;
$bulk_track_insert[$key_insert]['userid'] = 0;
$bulk_track_insert[$key_insert]['tag'] = 'vmusic';
$bulk_track_insert[$key_insert]['genre'] = 1;
$bulk_track_insert[$key_insert]['download_hash'] = md5($track->id);
$bulk_track_insert[$key_insert]['time'] = time();
$bulk_track_insert[$key_insert]['link_keeng'] = isset($track->url) ? $track->url : null;
$bulk_track_insert[$key_insert]['id_keeng'] = isset($track->id) ? $track->id : null;
$bulk_track_insert[$key_insert]['singer_list'] = isset($track->singer) ? $track->singer : null;
$bulk_track_insert[$key_insert]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
$bulk_track_insert[$key_insert]['src_thumbnail'] = isset($track->image) ? $track->image : null;
$bulk_track_insert[$key_insert]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
$bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
$key_insert++;
}
}
}
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
echo "\nKhông lấy được bài hát nào.";
}
// Ghi log
Crontjob::create(['message' => 'KeengCrawlerTrackCommand is called at ' . Carbon::now()]);
}
}
app/Console/Commands/Nct/NctCrawlerTrackCommand.php
......@@ -76,7 +76,9 @@ class NctCrawlerTrackCommand extends Command
// Cập nhật
$track_db = Track::where('title', $track['title'])->take(1)->get();
if (isset($track_db[0])) {
$track_db[0]->track_duration = $duration;
if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = $duration;
}
$track_db[0]->id_nct = $track['id_nct'];
$track_db[0]->id_nct_str = $track['id_nct_str'];
$track_db[0]->link_nct = $link;
......
app/Console/Commands/NhacVn/NhacVnCrawlerTrackCommand.php
......@@ -72,7 +72,9 @@ class NhacVnCrawlerTrackCommand extends Command
if (isset($track_db[0])) {
$track_db[0]->id_nhacvn = $track['id_nhacvn'];
$track_db[0]->link_nhacvn = $track['link'];
$track_db[0]->singer_list = $track['singer'];
if (empty($track_db[0]->singer_list)) {
$track_db[0]->singer_list = $track['singer'];
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
......
app/Console/Commands/Zing/ZingCrawlerNewRealeaseCommand.php
......@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
......@@ -69,25 +70,37 @@ class ZingCrawlerNewRealeaseCommand extends Command
$artists = isset($track->artists) ? $track->artists : [];
if (is_array($artists) && count($artists) > 0) {
$artist_list = collect($artists)->map(function ($item){ return $item->name; })->all();
$singer_list = implode(',', $artist_list);
$singer_list = implode('|', $artist_list);
}
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) {
$bulk_track_update[$key_update]['title'] = isset($track->title) ? $track->title : 'unknown-title';
$bulk_track_update[$key_update]['slug'] = isset($track->alias) ? $track->alias : null;
$bulk_track_update[$key_update]['userid'] = 0;
$bulk_track_update[$key_update]['tag'] = 'vmusic';
$bulk_track_update[$key_update]['genre'] = 1;
$bulk_track_update[$key_update]['download_hash'] = md5($track->alias);
$bulk_track_update[$key_update]['time'] = time();
$bulk_track_update[$key_update]['link'] = isset($track->link) ? $domain . $track->link : null;
$bulk_track_update[$key_update]['id_zing'] = isset($track->id) ? $track->id : null;
$bulk_track_update[$key_update]['singer_list'] = $singer_list;
$bulk_track_update[$key_update]['track_duration'] = isset($track->duration) ? $track->duration : 0;
$bulk_track_update[$key_update]['src_thumbnail'] = isset($track->thumbnail) ? $track->thumbnail : null;
$bulk_track_update[$key_update]['src_thumbnail_medium'] = isset($track->thumbnail_medium) ? $track->thumbnail_medium : null;
$bulk_track_update[$key_update]['link_mv'] = isset($track->mv_link) ? $domain . $track->mv_link : null;
$track_db = Track::where('title', $track->title)->take(1)->get();
if (isset($track_db[0])) {
if (empty($track_db[0]->slug)) {
$track_db[0]->slug = isset($track->alias) ? $track->alias : null;
}
if (empty($track_db[0]->download_hash)) {
$track_db[0]->download_hash = md5($track->alias);
}
if (empty($track_db[0]->time)) {
$track_db[0]->time = time();
}
if (empty($track_db[0]->link)) {
$track_db[0]->link = isset($track->link) ? $domain . $track->link : null;
}
if (empty($track_db[0]->id_zing)) {
$track_db[0]->id_zing = isset($track->id) ? $track->id : null;
}
if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = isset($track->duration) ? $track->duration : 0;
}
if (empty($track_db[0]->link_mv)) {
$track_db[0]->link_mv = isset($track->mv_link) ? $domain . $track->mv_link : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
......@@ -112,10 +125,10 @@ class ZingCrawlerNewRealeaseCommand extends Command
}
}
echo "\nSố bài hát insert: " . count($bulk_track_insert);
echo "\nSố bài hát update: " . count($bulk_track_update);
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if (count($bulk_track_insert) > 0) {
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
......
app/Console/Commands/Zing/ZingCrawlerTrackCommand.php
......@@ -4,6 +4,7 @@ namespace App\Console\Commands\Zing;
use App\Helpers\Constants;
use App\Models\Crontjob;
use App\Models\Track;
use App\Repositories\TrackRepository;
use Carbon\Carbon;
use Illuminate\Console\Command;
......@@ -69,25 +70,37 @@ class ZingCrawlerTrackCommand extends Command
$artists = isset($track->artists) ? $track->artists : [];
if (is_array($artists) && count($artists) > 0) {
$artist_list = collect($artists)->map(function ($item){ return $item->name; })->all();
$singer_list = implode(',', $artist_list);
$singer_list = implode('|', $artist_list);
}
// Xử lý insert/update dựa vào tên bài hát
if (is_array($inserted) && count($inserted) > 0 && in_array($track->title, $inserted)) {
$bulk_track_update[$key_update]['title'] = isset($track->title) ? $track->title : 'unknown-title';
$bulk_track_update[$key_update]['slug'] = isset($track->alias) ? $track->alias : null;
$bulk_track_update[$key_update]['userid'] = 0;
$bulk_track_update[$key_update]['tag'] = 'vmusic';
$bulk_track_update[$key_update]['genre'] = 1;
$bulk_track_update[$key_update]['download_hash'] = md5($track->alias);
$bulk_track_update[$key_update]['time'] = time();
$bulk_track_update[$key_update]['link'] = isset($track->link) ? $domain . $track->link : null;
$bulk_track_update[$key_update]['id_zing'] = isset($track->id) ? $track->id : null;
$bulk_track_update[$key_update]['singer_list'] = $singer_list;
$bulk_track_update[$key_update]['track_duration'] = isset($track->duration) ? $track->duration : 0;
$bulk_track_update[$key_update]['src_thumbnail'] = isset($track->thumbnail) ? $track->thumbnail : null;
$bulk_track_update[$key_update]['src_thumbnail_medium'] = isset($track->thumbnail_medium) ? $track->thumbnail_medium : null;
$bulk_track_update[$key_update]['link_mv'] = isset($track->mv_link) ? $domain . $track->mv_link : null;
$track_db = Track::where('title', $track->title)->take(1)->get();
if (isset($track_db[0])) {
if (empty($track_db[0]->slug)) {
$track_db[0]->slug = isset($track->alias) ? $track->alias : null;
}
if (empty($track_db[0]->download_hash)) {
$track_db[0]->download_hash = md5($track->alias);
}
if (empty($track_db[0]->time)) {
$track_db[0]->time = time();
}
if (empty($track_db[0]->link)) {
$track_db[0]->link = isset($track->link) ? $domain . $track->link : null;
}
if (empty($track_db[0]->id_zing)) {
$track_db[0]->id_zing = isset($track->id) ? $track->id : null;
}
if (empty($track_db[0]->track_duration)) {
$track_db[0]->track_duration = isset($track->duration) ? $track->duration : 0;
}
if (empty($track_db[0]->link_mv)) {
$track_db[0]->link_mv = isset($track->mv_link) ? $domain . $track->mv_link : null;
}
$track_db[0]->crawler_at = Carbon::now();
$track_db[0]->save();
}
$key_update++;
} else {
......@@ -112,10 +125,10 @@ class ZingCrawlerTrackCommand extends Command
}
}
echo "\nSố bài hát insert: " . count($bulk_track_insert);
echo "\nSố bài hát update: " . count($bulk_track_update);
echo "\nSố bài hát insert: " . $key_insert;
echo "\nSố bài hát update: " . $key_update;
if (count($bulk_track_insert) > 0) {
if ($key_insert > 0) {
DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
echo "\nInsert thành công.";
} else {
......
app/Console/Kernel.php
......@@ -3,6 +3,7 @@
namespace App\Console;
use App\Console\Commands\CrontjobCommand;
use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
use App\Console\Commands\Nct\NctCrawlerTrackCommand;
use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
use App\Console\Commands\Zing\ZingCrawlerNewRealeaseCommand;
......@@ -22,10 +23,10 @@ class Kernel extends ConsoleKernel
CrontjobCommand::class,
ZingCrawlerTrackCommand::class,
ZingCrawlerNewRealeaseCommand::class,
CrawlerTrackImageCommand::class,
NctCrawlerTrackCommand::class,
NhacVnCrawlerTrackCommand::class,
KeengCrawlerTrackCommand::class,
CrawlerTrackImageCommand::class,
];
......@@ -37,19 +38,18 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
// $schedule->command('inspire')
// ->hourly();
$schedule->command('crontjob:test')->hourly();
$schedule->command('zing:get-track')->everyThirtyMinutes();
$schedule->command('zing:get-new-release-track')->hourlyAt(5);
$schedule->command('crawler:get-track-image')->hourlyAt(10);
$schedule->command('nct:get-track')->hourlyAt(10);
$schedule->command('nhacvn:get-track')->hourlyAt(15);
$schedule->command('nct:get-track')->hourlyAt(25);
$schedule->command('keeng:get-track')->hourlyAt(20);
$schedule->command('nhacvn:get-track')->hourlyAt(35);
$schedule->command('crawler:get-track-image')->hourlyAt(35);
$schedule->command('queue:work --once --timeout=120')->everyMinute()->withoutOverlapping();
}
......
app/Helpers/Functions.php
......@@ -720,148 +720,32 @@ if (!function_exists('getUrlArtistByPage')) {
}
}
if (!function_exists('read_money')) {
/**
* Hàm đọc số tiền trên hóa đơn thành chuỗi
* doanpv - 14/06/2019
*
* @param $money
* @param string $currency_type
*
* @return string
*/
function read_money($money, $currency_type='1')
{
switch (strtoupper($currency_type)) {
case '1': // VND
$unit_name = 'đồng';
$sub_unit_name = 'xu';
$money = round($money, 0);
break;
case '2': // USD
$unit_name = 'Đô la Mỹ';
$sub_unit_name = 'cent';
$money = round($money, 2);
break;
case '3': // EUR
$unit_name = 'Ơ Rô';
$sub_unit_name = 'cent';
$money = round($money, 2);
break;
default:
$unit_name = 'đồng';
$sub_unit_name = 'xu';
$money = round($money, 0);
}
if ($money == 0) {
return 'Không ' . $unit_name . '.';
} elseif ($money < 0) {
return './.';
}
$number = [
1 => 'một',
2 => 'hai',
3 => 'ba',
4 => 'bốn',
5 => 'năm',
6 => 'sáu',
7 => 'bảy',
8 => 'tám',
9 => 'chín',
];
if ($money <= 2147483647) {
$odd = intval(($money - intval($money))*100);
$even = intval(trim($money));
$even = str_repeat(' ', 15 - strlen($even)) . $even;
$thousands_of_billions = intval(substr($even, 0, 3));
$billions = intval(substr($even, 3, 3));
$millions = intval(substr($even, 6, 3));
$thousands = intval(substr($even, 9, 3));
$units = intval(substr($even, 12, 3));
} else {
$odd = floatval(($money - floatval($money))*100);
$even = floatval(trim($money));
$even = str_repeat(' ', 15 - strlen($even)) . $even;
$thousands_of_billions = floatval(substr($even, 0, 3));
$billions = floatval(substr($even, 3, 3));
$millions = floatval(substr($even, 6, 3));
$thousands = floatval(substr($even, 9, 3));
$units = floatval(substr($even, 12, 3));
}
if ($thousands_of_billions == 0 && $billions == 0 && $millions == 0 && $thousands == 0 && $units == 0) {
$result = 'Không ' . $unit_name . ' ';
$i = 5;
} else {
$result = '';
$i = 0;
}
while ($i <= 5) {
switch ($i) {
case 0:
$temp = $thousands_of_billions;
$name = 'nghìn tỷ'; break;
case 1:
$temp = $billions;
$name = 'tỷ'; break;
case 2:
$temp = $millions;
$name = 'triệu'; break;
case 3:
$temp = $thousands;
$name = 'nghìn'; break;
case 4:
$temp = $units;
$name = $unit_name; break;
case 5:
$temp = $odd;
$name = $sub_unit_name; break;
}
if ($temp != 0) {
$hundred = intval($temp/100);
$ten = intval(($temp - $hundred*100)/10);
$unit = intval(($temp - $hundred * 100) - $ten*10);
if (substr($result, strlen($result) - 3) == ' ') {
$result = substr($result, strlen($result) - 3);
}
$result .= ((strlen($result) == 0) ? '' : ', ') . (($hundred != 0) ? $number[$hundred] . ' trăm ' : '');
if (!function_exists('cURL')) {
function cURL($url) {
$ch = curl_init();
if ($ten == 0 && $hundred != 0 && $unit != 0) {
$result .= 'lẻ ';
} else {
if($ten != 0) {
$result .= (($ten != 0 && $ten != 1) ? $number[$ten] . ' mươi ' : 'mười ');
}
}
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'GET');
if ($ten != 0 && $unit == 5) {
$result .= 'lăm ' . $name . ' ';
} else {
if ($ten > 1 && $unit == 1) {
$result .= 'mốt ' . $name . ' ';
} else {
$result .= (($unit != 0) ? $number[$unit] . ' ' . $name : $name) . ' ';
}
}
} else {
$result .= (($i == 4) ? $unit_name . '': '');
}
curl_setopt($ch, CURLOPT_ENCODING, 'gzip, deflate');
$i = $i + 1;
}
//$headers = array();
//$headers[] = 'Authority: mp3.zing.vn';
//$headers[] = 'Cache-Control: max-age=0';
//$headers[] = 'Upgrade-Insecure-Requests: 1';
//$headers[] = 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36';
//$headers[] = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3';
//$headers[] = 'Accept-Encoding: gzip, deflate, br';
//$headers[] = 'Accept-Language: vi-VN,vi;q=0.9,en-US;q=0.8,en;q=0.7,ja;q=0.6';
//curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
if ($odd == 0 && $currency_type == '1') {
$result .= ((substr($result, strlen($result) - 3) == ' ') ? '' : ' ') . 'chẵn';
$result = curl_exec($ch);
if (curl_errno($ch)) {
return false;
}
return ucfirst(str_replace(' ,', ',', $result)) . './.';
curl_close ($ch);
return $result;
}
}
......
config/api.php
......@@ -79,6 +79,12 @@ return [
'ranking20' => 'https://nhac.vn/bang-xep-hang-bai-hat-viet-nam-bxdE'
],
'keeng' => [
'domain' => 'http://keeng.vn/',
'ranking20' => 'http://keeng.vn/bang-xep-hang/song/viet-nam',
'api_get_rank' => 'http://vip.service.keeng.vn:8080/KeengWSRestful//ws/common/getRankDetail?item_type=1&rank_type=50'
],
];
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment