dọn code, thiết lập timeout

app/Console/Commands/CrawlerTrackImageCommand.php

@@ -36,6 +36,7 @@ class CrawlerTrackImageCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 
@@ -46,7 +47,7 @@ class CrawlerTrackImageCommand extends Command
      */
     public function handle()
     {
-        $tracks = $this->trackRepository->getTrackImageEmpty();
+        $tracks = $this->trackRepository->getTrackImageEmpty(200);
 
         echo "\nTổng số bài hát cần lấy ảnh về: " . count($tracks);
 

app/Console/Commands/CrawlerTrackListenCommand.php

@@ -35,6 +35,7 @@ class CrawlerTrackListenCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 
@@ -90,6 +91,12 @@ class CrawlerTrackListenCommand extends Command
                                 ['today_nct' => $listen, 'nct_crawler_at' => Carbon::now()]
                             );
                             break;
+                        case 'keeng':
+                            CrawlerListen::updateOrCreate(
+                                ['track_id' => $track_id],
+                                ['today_keeng' => $listen, 'keeng_crawler_at' => Carbon::now()]
+                            );
+                            break;
                     }
                 }
             }
@@ -134,6 +141,18 @@ class CrawlerTrackListenCommand extends Command
                 }
 
                 break;
+            case 'keeng':
+                $url = getKeengUrlGetCounter($track['id_keeng']);
+                $curl = cURL($url);
+                $response = json_decode($curl);
+
+                if (isset($response->data)) {
+                    if (isset($response->data->listen_no) && $response->data->listen_no > 0) {
+                        $listen = $response->data->listen_no;
+                    }
+                }
+
+                break;
         }
 
         return $listen;

app/Console/Commands/Keeng/KeengCrawlerByCateCommand.php

+<?php
+
+namespace App\Console\Commands\Keeng;
+
+use App\Helpers\Constants;
+use App\Models\Crontjob;
+use App\Models\Track;
+use App\Repositories\TrackRepository;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class KeengCrawlerByCateCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'keeng:get-track-by-cate';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Lấy những bài hát mới phát hành';
+
+    protected $trackRepository;
+
+    /**
+     * Create a new command instance.
+     *
+     * KeengCrawlerByCateCommand constructor.
+     * @param TrackRepository $trackRepository
+     */
+    public function __construct(TrackRepository $trackRepository)
+    {
+        parent::__construct();
+        ini_set('max_execution_time', -1);
+        $this->trackRepository = $trackRepository;
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        echo "\nBắt đầu xử lý lấy 100 bài hát theo từng trang từ Keeng.";
+
+        // Lấy mảng ID của Zing những bài hát đã insert
+        $inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
+
+        // Lấy ds BXH về
+        $domain = config('api.keeng.domain');
+        $tracks = [];
+        $index = 0;
+        for ($i = 1; $i <= 10; $i++) {
+            $url = getKeengUrlCategoryByPopViet($i);
+            $curl = cURL($url);
+            $data = json_decode($curl);
+            if (isset($data->data)) {
+                foreach ($data->data as $key => $track) {
+                    $tracks[$index]['title'] = isset($track->name) ? $track->name : 'unknown-title';
+                    $tracks[$index]['slug'] = isset($track->slug) ? $track->slug : null;
+                    $tracks[$index]['userid'] = 0;
+                    $tracks[$index]['tag'] = 'vmusic';
+                    $tracks[$index]['genre'] = 1;
+                    $tracks[$index]['download_hash'] = md5($track->id);
+                    $tracks[$index]['time'] = time();
+                    $tracks[$index]['link_keeng'] = isset($track->url) ? $track->url : null;
+                    $tracks[$index]['id_keeng'] = isset($track->identify) ? $track->identify : null;
+                    $tracks[$index]['singer_list'] = isset($track->singer) ? $track->singer : null;
+                    $tracks[$index]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
+                    $tracks[$index]['src_thumbnail'] = isset($track->image) ? $track->image : null;
+                    $tracks[$index]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
+                    $index++;
+                }
+            }
+        }
+
+        $bulk_track_insert = [];
+        $bulk_track_update = [];
+        $key_insert = 0;
+        $key_update = 0;
+        foreach ($tracks as $key => $track) {
+            // Xử lý insert/update dựa vào tên bài hát
+            if (is_array($inserted) && count($inserted) > 0 && in_array($track['title'], $inserted)) {
+                $track_db = Track::where('title', $track['title'])->take(1)->get();
+                if (isset($track_db[0])) {
+                    $track_db[0]->link_keeng = $track['link_keeng'];
+                    $track_db[0]->id_keeng = $track['id_keeng'];
+                    if (empty($track_db[0]->singer_list)) {
+                        $track_db[0]->singer_list = $track['singer_list'];
+                    }
+                    if (empty($track_db[0]->composer_list)) {
+                        $track_db[0]->composer_list = $track['composer_list'];
+                    }
+                    $track_db[0]->crawler_at = Carbon::now();
+                    $track_db[0]->save();
+                }
+
+                $key_update++;
+            } else {
+                $bulk_track_insert[$key_insert]['title'] = $track['title'];
+                $bulk_track_insert[$key_insert]['slug'] = $track['slug'];
+                $bulk_track_insert[$key_insert]['userid'] = 0;
+                $bulk_track_insert[$key_insert]['tag'] = 'vmusic';
+                $bulk_track_insert[$key_insert]['genre'] = 1;
+                $bulk_track_insert[$key_insert]['download_hash'] = md5($track['download_hash']);
+                $bulk_track_insert[$key_insert]['time'] = time();
+                $bulk_track_insert[$key_insert]['link_keeng'] = $track['link_keeng'];
+                $bulk_track_insert[$key_insert]['id_keeng'] = $track['id_keeng'];
+                $bulk_track_insert[$key_insert]['singer_list'] = $track['singer_list'];
+                $bulk_track_insert[$key_insert]['composer_list'] = $track['composer_list'];
+                $bulk_track_insert[$key_insert]['src_thumbnail'] = $track['src_thumbnail'];
+                $bulk_track_insert[$key_insert]['src_thumbnail_medium'] = $track['src_thumbnail_medium'];
+                $bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
+
+                $key_insert++;
+            }
+        }
+
+        echo "\nSố bài hát insert: " . $key_insert;
+        echo "\nSố bài hát update: " . $key_update;
+
+        if ($key_insert > 0) {
+            DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
+            echo "\nInsert thành công.";
+        } else {
+            echo "\nKhông lấy được bài hát nào.";
+        }
+
+        // Ghi log
+        Crontjob::create(['message' => 'KeengCrawlerByCateCommand is called at ' . Carbon::now()]);
+    }
+}

app/Console/Commands/Keeng/KeengCrawlerNewRealeaseCommand.php

+<?php
+
+namespace App\Console\Commands\Keeng;
+
+use App\Helpers\Constants;
+use App\Models\Crontjob;
+use App\Models\Track;
+use App\Repositories\TrackRepository;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class KeengCrawlerNewRealeaseCommand extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'keeng:get-new-release-track';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Lấy những bài hát mới phát hành';
+
+    protected $trackRepository;
+
+    /**
+     * Create a new command instance.
+     *
+     * KeengCrawlerNewRealeaseCommand constructor.
+     * @param TrackRepository $trackRepository
+     */
+    public function __construct(TrackRepository $trackRepository)
+    {
+        parent::__construct();
+        ini_set('max_execution_time', -1);
+        $this->trackRepository = $trackRepository;
+    }
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        echo "\nBắt đầu xử lý lấy 100 bài hát theo từng trang từ Keeng.";
+
+        // Lấy mảng ID của Zing những bài hát đã insert
+        $inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
+
+        // Lấy ds BXH về
+        $domain = config('api.keeng.domain');
+        $tracks = [];
+        $index = 0;
+        for ($i = 1; $i <= 10; $i++) {
+            $url = getKeengUrlNewRealease($i);
+            $curl = cURL($url);
+            $data = json_decode($curl);
+            if (isset($data->data)) {
+                foreach ($data->data as $key => $track) {
+                    $tracks[$index]['title'] = isset($track->name) ? $track->name : 'unknown-title';
+                    $tracks[$index]['slug'] = isset($track->slug) ? $track->slug : null;
+                    $tracks[$index]['userid'] = 0;
+                    $tracks[$index]['tag'] = 'vmusic';
+                    $tracks[$index]['genre'] = 1;
+                    $tracks[$index]['download_hash'] = md5($track->id);
+                    $tracks[$index]['time'] = time();
+                    $tracks[$index]['link_keeng'] = isset($track->url) ? $track->url : null;
+                    $tracks[$index]['id_keeng'] = isset($track->identify) ? $track->identify : null;
+                    $tracks[$index]['singer_list'] = isset($track->singer) ? $track->singer : null;
+                    $tracks[$index]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
+                    $tracks[$index]['src_thumbnail'] = isset($track->image) ? $track->image : null;
+                    $tracks[$index]['src_thumbnail_medium'] = isset($track->image310) ? $track->image310 : null;
+                    $index++;
+                }
+            }
+        }
+
+        $bulk_track_insert = [];
+        $bulk_track_update = [];
+        $key_insert = 0;
+        $key_update = 0;
+        foreach ($tracks as $key => $track) {
+            // Xử lý insert/update dựa vào tên bài hát
+            if (is_array($inserted) && count($inserted) > 0 && in_array($track['title'], $inserted)) {
+                $track_db = Track::where('title', $track['title'])->take(1)->get();
+                if (isset($track_db[0])) {
+                    $track_db[0]->link_keeng = $track['link_keeng'];
+                    $track_db[0]->id_keeng = $track['id_keeng'];
+                    if (empty($track_db[0]->singer_list)) {
+                        $track_db[0]->singer_list = $track['singer_list'];
+                    }
+                    if (empty($track_db[0]->composer_list)) {
+                        $track_db[0]->composer_list = $track['composer_list'];
+                    }
+                    $track_db[0]->crawler_at = Carbon::now();
+                    $track_db[0]->save();
+                }
+
+                $key_update++;
+            } else {
+                $bulk_track_insert[$key_insert]['title'] = $track['title'];
+                $bulk_track_insert[$key_insert]['slug'] = $track['slug'];
+                $bulk_track_insert[$key_insert]['userid'] = 0;
+                $bulk_track_insert[$key_insert]['tag'] = 'vmusic';
+                $bulk_track_insert[$key_insert]['genre'] = 1;
+                $bulk_track_insert[$key_insert]['download_hash'] = md5($track['download_hash']);
+                $bulk_track_insert[$key_insert]['time'] = time();
+                $bulk_track_insert[$key_insert]['link_keeng'] = $track['link_keeng'];
+                $bulk_track_insert[$key_insert]['id_keeng'] = $track['id_keeng'];
+                $bulk_track_insert[$key_insert]['singer_list'] = $track['singer_list'];
+                $bulk_track_insert[$key_insert]['composer_list'] = $track['composer_list'];
+                $bulk_track_insert[$key_insert]['src_thumbnail'] = $track['src_thumbnail'];
+                $bulk_track_insert[$key_insert]['src_thumbnail_medium'] = $track['src_thumbnail_medium'];
+                $bulk_track_insert[$key_insert]['crawler_at'] = Carbon::now();
+
+                $key_insert++;
+            }
+        }
+
+        echo "\nSố bài hát insert: " . $key_insert;
+        echo "\nSố bài hát update: " . $key_update;
+
+        if ($key_insert > 0) {
+            DB::table(Constants::TABLE_TRACKS)->insert($bulk_track_insert);
+            echo "\nInsert thành công.";
+        } else {
+            echo "\nKhông lấy được bài hát nào.";
+        }
+
+        // Ghi log
+        Crontjob::create(['message' => 'KeengCrawlerNewRealeaseCommand is called at ' . Carbon::now()]);
+    }
+}

app/Console/Commands/Keeng/KeengCrawlerTrackCommand.php

@@ -37,6 +37,7 @@ class KeengCrawlerTrackCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 
@@ -47,7 +48,7 @@ class KeengCrawlerTrackCommand extends Command
      */
     public function handle()
     {
-        echo "\nBắt đầu xử lý lấy top 100 bài hát từ Keeng.";
+        echo "\nBắt đầu xử lý lấy top bài hát từ Keeng.";
 
         // Lấy mảng ID của Zing những bài hát đã insert
         $inserted = $this->trackRepository->getTrackInserted('title', 'original', []);
@@ -59,12 +60,6 @@ class KeengCrawlerTrackCommand extends Command
         $data = json_decode($curl);
         $tracks = $data->data;
 
-        //$tracks = cURL($url);
-
-        //print_r($tracks);
-
-        //return false;
-
         $bulk_track_insert = [];
         $bulk_track_update = [];
         $key_insert = 0;
@@ -76,7 +71,7 @@ class KeengCrawlerTrackCommand extends Command
                     $track_db = Track::where('title', $track->name)->take(1)->get();
                     if (isset($track_db[0])) {
                         $track_db[0]->link_keeng = isset($track->url) ? $track->url : null;
-                        $track_db[0]->id_keeng = isset($track->id) ? $track->id : null;
+                        $track_db[0]->id_keeng = isset($track->identify) ? $track->identify : null;
                         if (empty($track_db[0]->singer_list)) {
                             $track_db[0]->singer_list = isset($track->singer) ? $track->singer : null;
                         }
@@ -97,7 +92,7 @@ class KeengCrawlerTrackCommand extends Command
                     $bulk_track_insert[$key_insert]['download_hash'] = md5($track->id);
                     $bulk_track_insert[$key_insert]['time'] = time();
                     $bulk_track_insert[$key_insert]['link_keeng'] = isset($track->url) ? $track->url : null;
-                    $bulk_track_insert[$key_insert]['id_keeng'] = isset($track->id) ? $track->id : null;
+                    $bulk_track_insert[$key_insert]['id_keeng'] = isset($track->identify) ? $track->identify : null;
                     $bulk_track_insert[$key_insert]['singer_list'] = isset($track->singer) ? $track->singer : null;
                     $bulk_track_insert[$key_insert]['composer_list'] = isset($track->info_extra->author_name) ? $track->info_extra->author_name : null;
                     $bulk_track_insert[$key_insert]['src_thumbnail'] = isset($track->image) ? $track->image : null;

app/Console/Commands/Nct/NctCrawlerTrackCommand.php

@@ -37,6 +37,7 @@ class NctCrawlerTrackCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 

app/Console/Commands/NhacVn/NhacVnCrawlerTrackCommand.php

@@ -37,6 +37,7 @@ class NhacVnCrawlerTrackCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 

app/Console/Commands/Zing/ZingCrawlerNewRealeaseCommand.php

@@ -37,6 +37,7 @@ class ZingCrawlerNewRealeaseCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 

app/Console/Commands/Zing/ZingCrawlerTrackCommand.php

@@ -37,6 +37,7 @@ class ZingCrawlerTrackCommand extends Command
     public function __construct(TrackRepository $trackRepository)
     {
         parent::__construct();
+        ini_set('max_execution_time', -1);
         $this->trackRepository = $trackRepository;
     }
 

app/Console/Kernel.php

@@ -4,6 +4,8 @@ namespace App\Console;
 
 use App\Console\Commands\CrawlerTrackListenCommand;
 use App\Console\Commands\CrontjobCommand;
+use App\Console\Commands\Keeng\KeengCrawlerByCateCommand;
+use App\Console\Commands\Keeng\KeengCrawlerNewRealeaseCommand;
 use App\Console\Commands\Keeng\KeengCrawlerTrackCommand;
 use App\Console\Commands\Nct\NctCrawlerTrackCommand;
 use App\Console\Commands\NhacVn\NhacVnCrawlerTrackCommand;
@@ -31,7 +33,9 @@ class Kernel extends ConsoleKernel
         CrawlerTrackImageCommand::class,
 
         //ZingCrawlerListenCommand::class,
-        CrawlerTrackListenCommand::class
+        CrawlerTrackListenCommand::class,
+        KeengCrawlerByCateCommand::class,
+        KeengCrawlerNewRealeaseCommand::class,
 
 
     ];

app/Helpers/Functions.php

@@ -227,3 +227,43 @@ if (!function_exists('getKeengUrlGetCounter')) {
         return $url;
     }
 }
+
+if (!function_exists('getKeengUrlCategoryByPopViet')) {
+    /**
+     * Hàm tạo URL để lấy thông tin bài hát theo chủ đề Pop Việt
+     * http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getCategoryDetail?page=100&num=100&slug=pop-viet&type=1
+     *
+     * @param $page
+     * @param $num
+     * @param $slug
+     * @param $type
+     * @return string
+     */
+    function getKeengUrlCategoryByPopViet($page = 1, $num = 100, $slug = 'pop-viet', $type = 1) {
+        $url = 'http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getCategoryDetail?';
+        $url .= 'page=' . $page;
+        $url .= '&num=' . $num;
+        $url .= '&slug=' . $slug;
+        $url .= '&type=' . $type;
+
+        return $url;
+    }
+}
+
+if (!function_exists('getKeengUrlNewRealease')) {
+    /**
+     * Hàm tạo URL để lấy thông tin bài hát theo chủ đề Pop Việt
+     * http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getListSongNewV4?page=1&num=40
+     *
+     * @param $page
+     * @param $num
+     * @return string
+     */
+    function getKeengUrlNewRealease($page = 1, $num = 100) {
+        $url = 'http://vip.service.keeng.vn:8080/KeengWSRestful///ws/common/getListSongNewV4?';
+        $url .= 'page=' . $page;
+        $url .= '&num=' . $num;
+
+        return $url;
+    }
+}

app/Repositories/TrackRepository.php

@@ -59,11 +59,11 @@ class TrackRepository extends BaseRepository
      *
      * @return array
      */
-    public function getTrackImageEmpty()
+    public function getTrackImageEmpty($limit=200)
     {
         $query = Track::select(['id', 'art', 'id_zing', 'id_nct', 'id_keeng', 'src_thumbnail_medium'])
             ->whereNull('art')
-            ->whereNotNull('src_thumbnail_medium');
+            ->whereNotNull('src_thumbnail_medium')->take($limit);
 
         return $query->get()->toArray();
     }