Execute PowerShell from Python


• January 11, 2019 • Leave a Comment

I originally wrote this to execute PowerShell scripts from a Flask web interface. You can now install Powershell on Linux. Crazy, ain’t it?

import subprocess
import datetime
import time
import os
import ast 
import json
# =======================================================
# Original name: lib_powershell.py
# =======================================================

class PowerShellProc():

    def __init__(self):
        self.pwrshell_path = r'C:\Windows\system32\WindowsPowerShell\v1.0\powershell.exe'
        self.station = r'\\repo\ps' # where the PS scripts live.

    def exec_pwrshell(self, pwr_script=None, script_path=None, args=[], ps_tuple_to_py=True, gm_flag=False, date2str=False):
        '''Out.exec_pwrshell('psscript.ps1' ,script_path=None, ps_tuple_to_py=True)
        script_path: overrides envrionment path.exists
        ps_tuple_to_py: Powershell tuples convert to dictionaries, set True to return a Python tuple.
        '''
        data = None
        exec_path = self.pwrshell_path
        
        if not os.path.exists(exec_path):
            raise IOError('Powershell not found at %s.' % exec_path)

        if not script_path:
            script_path = self.station
        
        if not os.path.exists(script_path):
            raise IOError('Script path not found at %s.' % script_path)

        if pwr_script:
            script_path = os.path.join(script_path, pwr_script)

        cmd = [exec_path, script_path]
        
        if args:
            if not isinstance(args, list):
                raise TypeError("Arguments must be passed as a list not %s." % str(type(args)))
            cmd.extend(args)

        proc = subprocess.Popen(cmd, 
                    shell=True,
                    stdin=subprocess.PIPE,
                    stdout=subprocess.PIPE,
                    )
        
        rtrn,perr = proc.communicate()
        rcode = proc.returncode

        #if rtrn:
            # Line directly below converts PowerShell Json to Python data structs.
            #data = self.ps_output_converstion(rtrn, tup=ps_tuple_to_py, gm=gm_flag, date2str=date2str)
        if perr:
            # Error string...
            data = json.dumps(perr.decode('utf-8','ignore'))
        else:
            #data = rtrn.decode('utf-8','ignore')
            data = self.ps_output_converstion(rtrn, tup=ps_tuple_to_py, gm=gm_flag, date2str=date2str)
        return rcode,data

    def ps_output_converstion(self, res, tup=True, gm=False, date2str=True):
        '''Converts Powershell string output to Python type'''
        
        res = res.decode('utf-8','ignore')
        
        if '\n' in res:
            res = res.replace('\n','')
        if '\r' in res:
            res = res.split('\r')[0]

        res = ast.literal_eval(res)
        
        # Data conversion if necessary.
        if type(res) == str:
            if 'date' in res.lower() or 'time' in res.lower():
                epoch_elem = elem[elem.find("(")+1:elem.rfind(")")]
                dt = self.ps_epoch_to_datetime(float(elem), gm=gm, date2str=date2str)
                res = dt
        if type(res) == list:
            for index, elem in enumerate(res):
                if 'date' in str(elem).lower() or 'date' in str(elem).lower():
                    epoch_elem = elem[elem.find("(")+1:elem.rfind(")")]
                    dt = self.ps_epoch_to_datetime(float(elem), gm=gm, date2str=date2str)
                    res.remove(elem)
                    res.insert(index,dt)
        if type(res) == dict:
            for field, value in res.items():
                if 'date' in str(value).lower():
                    epoch_elem = value[value.find("(")+1:value.rfind(")")]
                    dt = self.ps_epoch_to_datetime(float(epoch_elem), gm=gm, date2str=date2str)
                    res[field] = dt
            if tup:
                if 'Item1' in res:
                    l = [v for v in res.values()]
                    res = tuple(l)
        return res

    def ps_epoch_to_datetime(self, epoch_time, gm=False, date2str=False):
        '''Converts Powershell Date to Python datetime.'''
        if gm:
            struct = time.gmtime(epoch_time/1000.)
        else:
            struct = time.localtime(epoch_time/1000.)

        dt = datetime.datetime.fromtimestamp(time.mktime(struct))
        if date2str: dt = dt.ctime()
        return dt

    def py2json(self, data):
        return json.dumps(data) # This will convert datetime to string.

    def json2py(self, data):
        return json.loads(data)

'''
Examples of various data structures returned by a Powershell script.

>>> import lib_powershell
>>> ps = lib_powershell.PowerShellProc()
>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell',ps_tuple_to_py=False) # Tuple to dict.
(0, {'Item3': 1, 'Length': 3, 'Item1': 'Theresa Wilson', 'Item2': datetime.datetime(2018, 1, 1, 6, 0)})

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell')
(0, (1, 3, 'Theresa Wilson', datetime.datetime(2018, 1, 1, 6, 0)))

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell', gm_flag=True) # Mixed Tuple
(0, (1, 3, 'Theresa Wilson', datetime.datetime(2018, 1, 1, 11, 0)))

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell')
(0, {'Oregon': 'Salem', 'Washington': 'Olympia', 'California': 'Sacramento'})

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell') # Mixed array/arraylist
(0, ['a', 'b', 'c', 'd', 'e', 1, 2, 3])

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell') # Int
(0, 1)

>>> ps.exec_pwrshell(pwr_script='test.ps1',script_path=r'E:\pyprod\prod_pwrshell') # String
(0, 'Hello')

# NOTE: Tuple returned is (process_return_code, powershell_return_data)
'''

List or search all installed apps on Ubuntu 18.04 with Python.


• January 10, 2019 • Leave a Comment

I was bored… Of course this can be done easily in the terminal, but what’s the fun in that? This should work on any flavor of Debian/Ubuntu.

import os
import subprocess as sp
'''
Pipe to file in terminal:

$ flatpak list --app --show-details > ~/Documents/installed-flatpack.log
$ dpkg --get-selections > ~/Documents/installed-software.log

Not using Debian? Here's some others, including BSD Unix.
$ rpm -qa
$ yum list installed
$ pacman -Qi
$ pkg_version | less

Why Python when you have pipes, greps, and such?
Why not?
'''

def get_flatpak():
    cmd=['/usr/bin/flatpak','list','--app','--show-details']
    process=sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    stdout, stderr = process.communicate()
    return stdout.decode().replace('\t',' ').split('\n')

def get_apt():
    cmd=['/usr/bin/dpkg','--get-selections']
    process=sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE)
    stdout, stderr = process.communicate()
    return stdout.decode().replace('\t',' ').split('\n')
    
if __name__=="__main__":
    
    names_only = []
    flatpak = True
    apt = True
    search_string='gimp'
    
    if flatpak:
        pkgs = get_flatpak()
        for pkg in pkgs:
            if ' ' in pkg:
                names_only.append(pkg.split()[0])
                
    if apt:
        pkgs = get_apt()
        for pkg in pkgs:
            if ' ' in pkg:
                names_only.append(pkg.split()[0])
        
    for app in names_only:
        if search_string:
            if search_string not in app.lower():
                continue
        print(app)

Create Desktop App from WebApp.


• January 10, 2019 • Leave a Comment

Here’s a nifty tool for Windows, OS X, and Linux to convert web apps, like the online Amazon Kindle Reader, to a desktop app.

https://www.npmjs.com/package/nativefier

Running Steam Windows Games on Linux


• December 30, 2018 • Leave a Comment

I’m sure these will be hit or miss, but with a little work some Steam Windows games might run. If you want a unified gaming center on Linux with Steam for Windows support check out Lutris. Might be easier than pure Wine or Play On Linux. We will see.

Screenshot

and…

Watch Dogs 2 is running without issue on Linux!

Screenshot

Now getting the Nvidia drivers working at full steam is another matter. I am using 415 version with a GTX 1060. These can be installed from the Ubuntu PPA:

ppa:graphics-drivers/ppa

Here’s a nice write up on linuxconfig.org

Using Linux with iPhone the easy way.


• December 27, 2018 • Leave a Comment

Step one: Photos and video.

Most distros will mount the DCIM so you can access your photos, but I want to control them from the phone and select from there what I want to keep. So I setup a sync folder on my largest drive and setup Shotwell to watch its photos folder on the same drive. I then setup MEGA Sync, which is like Dropbox but you get 50gb free, to save to the sync folder. I wrote a small Python program to move files from the sync folder to the photos folder and set it as a task to run every 10 minutes. I turned off the camera sync in the MEGA iPhone app, otherwise it will move every photo you take. When I want photos in Shotwell I simply move the photo(s) to the MEGA app on the phone and let the cron task and Shotwell manage the files as they show up. It works great.

The code is simple, here’s an example:

#!/usr/bin/env python
import os
import shutil
 
def move_files(src_root, dest_root):

    dest_photo = os.path.join(dest_root,'Photo')
    dest_music = os.path.join(dest_root,'Music')
    dest_other = os.path.join(dest_root,'Documents')

    for root, dirs, files in os.walk(src_root):
        for _file in files:

            file_path = os.path.join(src_root, _file)

            if os.path.isfile(file_path):

                ext = os.path.splitext(_file.lower())[-1]

                if ext in ['.mov','.jpg','.png','.mp4']:
                    dest = os.path.join(dest_photo,_file)
                    shutil.copyfile(file_path,dest)
                    os.remove(file_path)
                elif ext in ['.mp3','.ogg','.flac']:
                    dest = os.path.join(dest_music,_file)
                    shutil.copyfile(file_path,dest)
                    os.remove(file_path)
                else:
                    dest = os.path.join(dest_other,_file)
                    shutil.copyfile(file_path,dest)
                    os.remove(file_path)
                    
if __name__ == "__main__":
    
    base_src = '/mnt/storage/MEGASync/'
    base_dest = '/mnt/storage'
    
    move_files(base_src, base_dest)

You could modify this script to isolate and move any file type to various folders, like if a Python file gets dropped into the Mega folder you could have the script move it to Code, or move text files to Writing. This Python code should also work on OS X and Windows. Make sure you change the paths to yours.

Add a directory named outgoing to the Megasync directory for anthing you want to pick up on the phone so it doesnt get moved. Anything there will be left alone since we are not traversing all sub directories.

Step two: Music.

Since you cannot manipulate the contents of the iTunes library on the phone I dumped Apple’s music player. I hated the ads anyway, plus I do not stream. I could install iTunes in Wine but I like Rhythmbox. I installed Flacbox on the iPhone and setup a Samba share on my music folder on my laptop. I logged into the share in Flacbox where I manage the music and playlists on my phone. I don’t mind managing phone and desktop playlists separately. Eitherway, my whole library on my machine is on the iPhone at my deposal without storing it there, unless I want to. I also researved my ip on my router so I never need to remount the share unless I change things.

Step three: App data

Install ifuse if it’s not installed.

sudo apt install ifuse

Create a folder in mnt named iPhone.

You will need the app id. Here is an example of an app named Number9 by a company named Magic Hat and how to mount its sandboxed folder:

sudo ifuse /mnt/iPhone –appid com.magichat.Number9

If successful you can browse the app’s files. This is what I am working on now. It’s a pain as I do not want to jailbreak my device and have to guess or connent to iTunes to get the ID. If there is a way on the phone I coundn’t find it.

If you need help with Samba I’m certain there are resources on the internet.

Other options?

Photos: Digikam will connect to the phone but it rarely worked for me. Alternatively Rapid Photo Downloader worked better but still hit or miss. It will also allow duplicates as will most photo managers.

Music: In Flacbox you can fire up its internal server and move music through a web browser. A little more work but it works. There are other players that will allow music imports, I think Vox will, but they are greedy.

Password Manager: If you keep Passwords in a manager you can use Keepass, but Enpass is the best. I have used it for years on Mac and now they have a Linux version that syncs with the iPhone version.

Note:

If you’re not keen to crontab in the terminal this tool makes life easy, and it works on Ubuntu 18.04. Gnome Scheduler

Compiled PDF List of the best Linux Software of 2018.


• December 23, 2018 • Leave a Comment

At least in my opinion. 🙂 Yes, it’s a PDF – I’m too lazy to sit here and format HTML.

Download as PDF

Anything you think should be here or if you need help finding something, leave a comment.

Review or Delete Duplicate Files with Python.


• December 22, 2018 • Leave a Comment

My music and photos were getting out of hand. Thought I would share.

'''
Author: C. Nichols, mohawke@gmail.com
Release: Dec. 2018

Simple script to review or remove duplicate files in a folder.
** Does not delete unless specifed.

TO RUN: See config:
    * Set dir        - path to check.
    * Set do_deletes - delete or review log mode.
    * Set log_name   - Log is unique to each run with datetime.
    * Run in terminal, cd to the folder you saved to script to and type:
        python3 dupe_killer.py
        OR
        python dupe_killer.py
'''
import os
import datetime
import hashlib

def chunk_reader(fobj, chunk_size=2048):
    """Generator that reads a file in chunks of bytes"""
    while True:
        chunk = fobj.read(chunk_size)
        if not chunk:
            return
        yield chunk

def get_hash(_file, chunk_size=2048, hash=hashlib.sha512):
    hashobj = hash()
    with open(_file, 'rb') as file_object:
        hashobj.update(file_object.read(chunk_size))
        ##for chunk in chunk_reader(file_object, chunk_size=chunk_size):
        ##    hashobj.update(chunk)
    _hash = hashobj.digest()
    return _hash

def get_ctime(_file):
    return os.path.getctime(_file)

def get_size(_file):
    return os.path.getsize(_file)

def collect_files(path, delete=False):
    unique = {}
    for root, dirs, files in os.walk(path):
        for _file in files:

            file_path = os.path.join(root, _file)

            if os.path.isfile(file_path):
                _sz = get_size(file_path)
                _ct = get_ctime(file_path)
                _hd = get_hash(file_path)
                key = '%s_%s' % (_hd,_sz)
                if key not in unique:
                    unique[key] = file_path
                    yield 'Unique,"%s","%s",%s,"%s"\n' % (_file,datetime.datetime.fromtimestamp(_ct),_sz,root)
                else:
                    if delete:
                        os.remove(file_path)
                    yield 'Removed,"%s","%s",%s,"%s"\n' % (_file,datetime.datetime.fromtimestamp(_ct),_sz,root)

if __name__ == '__main__':

    # ===================================
    # Config
    # ===================================
    '''
    Examples:
        dir = r'C:\Music'
        dir = 'C:/Music'
        dir = '/Music'
    '''

    dir = r'/home/mohawke/Music' # Set the path to the folder you wish to check.

    do_deletes = False # False will let you review the log before committing to removal.

    log_name = 'dupe_kill'

    # ====================================
    # No changes needed below this line.
    # ====================================
    log_file = '%s_%s.csv' % (log_name, datetime.datetime.now().strftime('%Y%m%d_%H%M%S'))

    with open(log_file,'w') as log:
        log.write('Stat,File,Created,Size,Path\n')
        for dupe in collect_files(dir, delete=do_deletes):
            log.write(dupe)

Extract CAB files with Python and Patool.


• December 21, 2018 • Leave a Comment

Install 7zip.

This assumes you are on Windows.

Add the 7zip path to the system path. In the searchbar type env and open the result. Click Environment Variables in the dialog. Add to PATH. Windows 10 just click add and browse to the folder 7zip installed to, i.e., C:\Program Files\7-zip. Otherwise, paste the path making sure it’s at the end of the line followed by a ;

With Python installed, I use 3.7, open the terminal. In the searchbar type cmd and open the result. In the terminal type: pip install patool

You are all set. The only thing I found, and this is likely to do with 7zip, is when extracting many CABS to a single destination it hangs. I did not test this using 7z.exe directly. Just create a folder for each CAB you extract and move the extracted files to a single folder.

Here’s one I wrote to extract all the crap in DirectX9 from MS separated by 32/64 bit.

#!/usr/bin/python3
import os
import shutil
import patoolib

def unpack_cabs(source,dest,extract=True):
    if not os.path.exists(dest):
        os.mkdir(dest)
        
    prg = patoolib.find_archive_program(('cab'), 'extract', program=r'C:\Program Files\7-Zip\7z.exe')
    if extract:
        for cab in os.listdir(source):
            cab = cab.strip().lower()
            if not cab.endswith('.cab'): continue
            sub_folder = os.path.join(dest,cab.split('.')[0])
            if not os.path.exists(sub_folder):
                os.mkdir(sub_folder)

            full_source = os.path.join(source.lower(),cab)
            print('extracting %s' % sub_folder) 
            patoolib.extract_archive(full_source, outdir=sub_folder, program=prg)
            
    tmpx86 = os.path.join(dest,'tmp', 'x86')
    tmpx64 = os.path.join(dest,'tmp', 'x64')
    if not os.path.exists(tmpx86):
        os.makedirs(tmpx86)
    if not os.path.exists(tmpx64):
        os.makedirs(tmpx64)
    
    arch={'x86':{},'x64':{}}
    filter = ['x3d','d3d','xau','xin','xact','xapo']
    for exfolder in os.listdir(dest):
        
        if 'tmp' in exfolder: continue
        
        d_type = exfolder.split('_')[-1]
        
        subs = os.path.join(dest,exfolder)
        
        for dll in os.listdir(subs):
            dll = dll.strip().lower()
            fname = dll.split('.')[0]
            if not dll.endswith('.dll'): continue
            if dll[:3] not in filter: continue

            if d_type == 'x86':
                print(d_type, os.path.join(subs.lower(),dll),'->',os.path.join(tmpx86.lower(),dll))
                shutil.copyfile(os.path.join(subs.lower(),dll),os.path.join(tmpx86.lower(),dll))
                if fname not in arch[d_type]:
                    arch[d_type][fname] = None
            
                arch[d_type][fname] = os.path.join(tmpx64.lower(),dll)

            if d_type == 'x64':
                print(d_type, os.path.join(subs.lower(),dll),'->',os.path.join(tmpx64.lower(),dll))
                shutil.copyfile(os.path.join(subs.lower(),dll),os.path.join(tmpx64.lower(),dll))
                if fname not in arch[d_type]:
                    arch[d_type][fname] = None
            
                arch[d_type][fname] = os.path.join(tmpx64.lower(),dll)
    return arch

Linux for Graphic Artists and Designers: A real world setup.


• December 15, 2018 • Leave a Comment

I know a lot of Linux and software bloggers write often about the “Photoshop killers this” and the “best Linux/open source that” but I want to add my two cents as a long time Linux and open source user – early nineties to present – who does graphic design. I’m also not gaining financially to pump out those types of articles. I have been writing about this stuff before it was popular and my previous site here had a good following for Linux tutorials and free software listings, even though I’m not the best writer. 🙂 I switched to WordPress a few years ago and rarely update these days since the internet is flooded with similar content, but from time to time I like to think about what I’m doing and how I can share my experiences.

Over the many years I have done graphics I have worked on BeOS, Linux, Mac, and Windows. I have software all over the place for all four OSes. On Windows I think Adobe used to be the best option by far, not that their software was super amazing as you can make art in PaintBrush if you really want to, but rather for the plug-ins, brushes, and ease of use. I swicthed to OS X as my main OS for a few years only using Linux for programming. Of course I ran out and bought Photoshop 7 so I could have the best on Mac, but I lost interest because the software really never changed over the years. Sure, they move things around, make things easier, but honestly there wasn’t a huge change and the software does what it always did.

So my Macbook Pro was finally deemed too old for Apple to support so I decided, considering the apps in the App store are really not any better than their open source cousins I’ve used over the years, that their model isn’t really for me just like Microsoft years earlier. I honestly don’t want to spend $5k to $7k for a decent machine or be controlled by any corporation. Apple is now like Microsoft, maybe worse at this point. I decided to buy a powerful laptop with 16gb ram, fast solid state drives, 12 core i7 latest gen. CPU, and mid-range Nvidia, and best of all, a 17″ display. I ended up with a System 76 machine to support Linux even more and considering the cost to rent Adobe, which is unfortunate because CS6 is the best of the product line ever, I don’t think I’ll miss being billed to work and play.

Actual specs-
Screen size: 17″
Ram: 16gb
CPU: Intel® Core™ i7-8750H CPU @ 2.20GHz × 12
Video: Intel® UHD Graphics 630 (Coffeelake 3×8 GT2) Nividia 1060
Disk 1: 250 gb solid state
Disk 2: 1 tb solid state.

Now you have a bit of background on me let’s get into my choice of Linux and desktop environment. I had the machine shipped with System 76’s Pop! OS, which is a tweaked Ubuntu 18.04 and Gnome 3. I hated it! The core, however, is solid and their drivers and driver support is tops. After giving LinuxMint, Deepin, and Manjaro a try I switched back to Pop and installed Budgie to make it more like OS X. I am not a fan of Gnome 3. Budgie gave me a little trouble but with a little tweaking I find it to be the best environment I’ve worked in for a while. Now that the OS is solid and secured it was time to install the software, and soon after I got a gig to do a bookcover, so away we go.

Inkscape
https://inkscape.org

For the bookcover I needed to create a scroll logo and graphic element. This choice was easy, Inkscape! I have nothing bad to say about Inkscape. It works and works well. For Vector work I have been using it on Windows, Mac, and Linux for years. It does the job of Adobe Illustrator so I never bought a copy. It is the most consistently good application in the open source world.

Gimp
https://www.gimp.org

Gimp worked well with a big image where Krita gave me some trouble. I paint in Krita at high res and had not had this problem before so it may be the latest version I used. I think Krita is as close to Photoshop as you’ll get in the opensource world and it is typically just fine, I won’t fault Krita, it’s really solid and under heavy development. Either way, Gimp did the job of adjusting the image to get it ready for print at 300dpi.

What you lose using these tools. Gimp is a bit of a learning curve from Photoshop, Krita not so much. The way I see it Gimp makes you work harder, or learn the non-automated way to get things done. If you need nice brushes and easier effects use Krita, if you don’t care either tool will do so pick the interface that suits you.

Plug-ins are basically non-existant. Gimp has a few really good plug-ins but finding them and finding versions that work with newer versions can be frustrating. This leads to my biggest complaint against open source, people create and abandon projects/web-sites often so unless you are willing to edit code you have to sometimes wait for others to find an interest in keeping things updated and available.

If you need Gimp plug-ins for versions < 2.8, some work on 2.10 for me, try the Gimp Plugin Registry archive (yes, it is dead as of this writing). Another option is to try other flavors. If you want Resynthisizer, G’Mic, etc. go here.

Other Gimp goodies, Gimp Paint Studio

For Windows users using Gimp 2.10: Gimp Extensions

Some updated plug-ins for Gimp 2.10 and G’Mic for Gimp and Krita

G’Mic
Multiple plug-ins
LayerFX

Scribus
https://www.scribus.net

I don’t do a lot of layout and for the little things I do I typically use LibreOffice Draw. For the book I found wrapping text painful in Draw so I installed the latest Scribus 1.5.5 SNV. It took me a day to figure it out, mainly wrapping text properly, but once I got it down I found this to be excellent software! I do not typically use PageMaker/InDesign so I cannot compare, but the software worked as expected and took only a day of learning and another half day to get everything put together. Book went to print and the customer was extremely happy. This tool will stay in my toolbox. I will admit I poked around for other layout programs for Linux. I was not thrilled with my findings, still you might want to try the alternatives.

I tried VivaDesigner. This is a commercial product and compared to the big boys the price was a bit high for a lower end alternative, strike one. The Linux installer was outdated and would not install on my machine, strike two. I installed the Windows version in Wine, seemed limited for a pay product, strike three.

I also tried Pagestream, another commercial offering. After installing a library it started right up but it seemed limited and dated. It could be a great product, I simply decided not to invest in learning it.

Scribus won simply because it’s full featured, not confusing, and free, but you do have options if Scribus isn’t what you want.

Business Side

Filezilla
https://filezilla-project.org

When all is said and done you need to get the product to the client. I use Filezilla as it’s the best FTP app IMO.

Now that the tools were picked and product has been delivered I’ll go into the administrative tools that I use that others fail to mention.

Kapow Time Tracking
https://gottcode.org/kapow

I like to track my time when I start working on a project so I can give detailed billing. This tool is one that I was going to write until I found Kapow. I was a little bummed it already existed. Oh well. This tool will track time on tasks in real-time. You could just write down start and end time but this is so convenient.

Planner
https://wiki.gnome.org/action/show/Apps/Planner?action=show&redirect=Planner

If you’re on larger/longer projects and/or working with teams Planner Project Management may be for you. I have it for this purpose to keep track of everything.

Osmo
http://clayo.org/osmo

Osmo is a nice personal task manager. It will keep you on track.

Mercurial-scm
https://www.mercurial-scm.org

For backups and revision I use Mercurial with TortoiseHG. Better safe than sorry.

Office, etc.

For invoicing I designed a simple template in LibreOffice and all other tools are stock Gnome/Ubuntu apps, like Geary for mail, Gnote, Gnome Calender, etc.

Misc.

Screen Ruler

GNOME Color Manager. Link to color info, the program is in the software repo like many of the others.

Font Manager

Birdfont Create your own fonts.

PDF of my complete software catalog. Download as PDF

In conclusion, I find this workflow to be just as powerful as the expensive alternatives, at least for me. If you do occasional freelance you cannot go wrong with these selections. The best part is that most are also available on Mac and Windows so if you’re not into Linux you can still use or try them freely.

The book can be found on Amazon here.

If you need assistance feel free to shoot me an email. Link under my profile up top. 😉

Python3 Gtk3 Thread Example


• August 20, 2018 • Comments Off on Python3 Gtk3 Thread Example

Quick Python3 Gtk3 app to execute Python scripts to test long running threads in Gtk3 app. Some things could be better, like the python paths collection and displaying the PID, but I was focused on running threads without freezing the GUI and it just kind of happened. Hope it’s useful to someone.

#!/usr/bin/python3
import sys
if sys.version_info[0] < 3:
    raise Exception("Python 3 or a more recent version is required.")

if 'linux' not in sys.platform:
    raise Exception("Linux is required.")

try:
    import gi
    gi.require_version('Gtk', '3.0')
    from gi.repository import Gtk, Gio, GLib, GObject
except:
    raise Exception("Missing 'gi' [Gtk 3].")

try:
    import psutil
except:
    raise Exception("Missing 'psutil', try sudo apt install python3-psutil.")

import os
import os.path
import fnmatch
import pickle
import threading
import time
import getpass
from subprocess import PIPE
'''
Find all Python executables on Linux machine.
Allow easy execution of Python scripts from GUI,
because I'm lazy...
'''
GObject.threads_init() 

class Worker(threading.Thread):
    def __init__(self, cmd, callback):
        threading.Thread.__init__(self)
        self.cmd = cmd
        self.callback = callback

    def run(self):
        if not self.cmd:
            _get_execs() # Get Python execs.
            GObject.idle_add(self.callback)
        else:
            msg,err = _process(self.cmd)
            GObject.idle_add(self.callback,[msg,err])

def app_load():
    p = '/home/%s/.pyexec' % getpass.getuser()
    if not os.path.exists(p):
        os.mkdir(p)
    p = os.path.join(p,'execs.bin')
    with open(p, 'rb') as stream:
        data = stream.read()
        execs = pickle.loads(data)
    return execs

def update_execs(data):
    p = '/home/%s/.pyexec' % getpass.getuser()
    if not os.path.exists(p):
        os.mkdir(p)
    p = os.path.join(p,'execs.bin')
    with open(p, 'wb') as output:
        output.write(data)

def load_user():
    user = None
    p = '/home/%s/.pyexec/user.bin' % getpass.getuser()
    if os.path.exists(p):
        with open(p, 'rb') as stream:
            data = stream.read()
            user = pickle.loads(data)
    return user

def save_user(data):
    p = '/home/%s/.pyexec' % getpass.getuser()
    if not os.path.exists(p):
        os.mkdir(p)
    p = os.path.join(p,'user.bin')
    data = pickle.dumps(data)
    with open(p, 'wb') as output:
        output.write(data)

def search(where='/', find='python*[!-config]'):
    for dirpath, dirnames, filenames in os.walk(where):
        if 'flatpak' in dirpath.lower(): continue
        if 'timeshift' in dirpath.lower(): continue
        if 'bin' not in dirpath.lower(): continue
        for filename in filenames:
            file_path = os.path.join(dirpath, filename)
            if not os.access(file_path, os.X_OK): continue
            if not fnmatch.fnmatch(filename, find): continue
            if not os.path.islink(file_path): continue
            yield file_path

def _get_execs():
    python_execs = [item for item in search()]
    dumped = pickle.dumps(python_execs)
    update_execs(dumped)

def _process(cmd):
    proc = psutil.Popen(cmd, stdout=PIPE, stderr=PIPE)
    print('NAME: %s PID = %s' % (proc.name(), str(proc.pid).split()[0]))
    msg, err = proc.communicate()
    return msg, err

class Executor(Gtk.Window):
    '''
    Main app window class.
    '''
    def __init__(self):
        Gtk.Window.__init__(self, title="Python Executor")

        self.set_border_width(10)
        self.set_default_size(400, 350)
        self.python_list = None
        self.proj_path = None
        self.script = None
        self.selected = None
        self.cmd = None
        self.msg = ''
        self.err = ''
        self.pid = ''
        self.versions = ['/usr/bin/python'] # placeholder

        # If the python list does not exist create, else populate versions.
        self.app_load_check() 

        # Load user settings.
        self.proj_path_default = "/home/%s/Documents/Python" % getpass.getuser()
        self.script_default = "script.py"
        self.cbox_index = 0
        user_settings = load_user()
        if user_settings:
            self.cbox_index, self.proj_path_default, self.script_default = user_settings 

        # Define header add buttons.
        header = Gtk.HeaderBar(title="Python Executor")
        header.props.show_close_button = True
        self.set_titlebar(header)

        # Button to refresh python list.
        refresh_button = Gtk.Button()
        icon_refresh = Gio.ThemedIcon(name="reload")
        image_refresh = Gtk.Image.new_from_gicon(icon_refresh, Gtk.IconSize.BUTTON)
        refresh_button.add(image_refresh)
        refresh_button.set_tooltip_text("Refresh environments")
        refresh_button.connect("clicked", self.on_refresh_clicked)
        header.pack_start(refresh_button)

        # Kill button.
        kill_button = Gtk.Button()
        icon_kill = Gio.ThemedIcon(name="process-stop")
        image_kill = Gtk.Image.new_from_gicon(icon_kill, Gtk.IconSize.BUTTON)
        kill_button.add(image_kill)
        kill_button.set_tooltip_text("Kill Process")
        kill_button.connect("clicked", self.on_kill_clicked)
        header.pack_start(kill_button)

        # Execute button.
        run_button = Gtk.Button()
        icon_run = Gio.ThemedIcon(name="run-build")
        image_run = Gtk.Image.new_from_gicon(icon_run, Gtk.IconSize.BUTTON)
        run_button.add(image_run)
        run_button.set_tooltip_text("Run")
        run_button.connect("clicked", self.on_run_clicked)
        header.pack_start(run_button)

        # Define layout.
        self.box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
        self.pybox = Gtk.Box(spacing=10)
        self.prjbox = Gtk.Box(spacing=10)
        self.scrbox = Gtk.Box(spacing=10)

        self.add(self.box)

        # Python execs.
        py_label = Gtk.Label()
        py_label.set_text("Python")
        py_label.set_justify(Gtk.Justification.LEFT)
        py_label.set_hexpand(False)

        self.py_store = Gtk.ListStore(str)
        for ver in self.versions:
            if isinstance(ver, str):
                self.py_store.append([ver])

        self.python_list = Gtk.ComboBox.new_with_model(self.py_store)
        self.python_list.connect("changed", self.on_changed)
        self.python_list.set_hexpand(False)
        renderer_text = Gtk.CellRendererText()
        self.python_list.pack_start(renderer_text, True)
        self.python_list.add_attribute(renderer_text, "text", 0)
        self.python_list.set_active(self.cbox_index) 

        self.pybox.pack_start(py_label, False, False, 0)
        self.pybox.pack_end(self.python_list, False, True, 0)
        self.box.pack_start(self.pybox, False, True, 0)

        # Project Path.
        prj_label = Gtk.Label()
        prj_label.set_text("Project ")
        prj_label.set_justify(Gtk.Justification.LEFT)
        prj_label.set_hexpand(False)

        self.proj_path = Gtk.Entry()
        self.proj_path.set_text(self.proj_path_default )
        self.proj_path.set_hexpand(True)

        self.prjbox.pack_start(prj_label, False, True, 0)
        self.prjbox.pack_end(self.proj_path, False, True, 0)
        self.box.pack_start(self.prjbox, False, True, 0)

        # Script
        scr_label = Gtk.Label()
        scr_label.set_text("Script   ")
        scr_label.set_justify(Gtk.Justification.LEFT)
        scr_label.set_hexpand(False)

        self.script = Gtk.Entry()
        self.script.set_text(self.script_default)
        self.script.set_hexpand(True)

        self.scrbox.pack_start(scr_label, False, True, 0)
        self.scrbox.pack_end(self.script, False, True, 0)
        self.box.pack_start(self.scrbox, False, True, 0)

        # Message area.
        scrolledwindow = Gtk.ScrolledWindow()
        scrolledwindow.set_hexpand(True)
        scrolledwindow.set_vexpand(True)

        self.message_view = Gtk.TextView()
        self.textbuffer = self.message_view.get_buffer()
        self.tag_err = self.textbuffer.create_tag('err', foreground="#FF0000")
        self.tag_msg = self.textbuffer.create_tag('msg', foreground="#FFFFFF")
        scrolledwindow.add(self.message_view) 
        self.box.pack_start(scrolledwindow, True, True, 0)

        self.show_all()

    def app_load_check(self):
        '''Fetch Python paths.'''
        p = '/home/%s/.pyexec' % getpass.getuser()
        if not os.path.exists(p):
            os.mkdir(p)
        p = os.path.join(p,'execs.bin')
        if not os.path.exists(p):
            thread = Worker(None, self.end_refresh)
            thread.start()
        else:
            self.versions = app_load()

    def on_refresh_clicked(self, widget):
        '''Refresh Python paths.'''
        thread = Worker(None, self.end_refresh)
        thread.start()

    def end_refresh(self):
        '''Thread callback.'''
        self.versions = app_load()
        for ver in self.versions:
            if ver not in self.py_store:
                if isinstance(ver, str):
                    self.py_store.append([ver])

    def on_changed(self, widget):
        '''Selected Python path.'''
        index = widget.get_active() 
        self.selected = widget.get_model()[index][0]

    def on_kill_clicked(self, widget):
        kill_msg = '\n* kill: Nothing to do.'
        end_iter = self.textbuffer.get_end_iter()
        for process in psutil.process_iter():
            if process.cmdline() == self.cmd:
                process.terminate()
                kill_msg = '\n* %s: %s killed' % (process.pid, process.cmdline())
                self.textbuffer.insert(end_iter, kill_msg)
                break

    def on_run_clicked(self, widget):
        '''Execute Python script.'''
        ok = True
        self.cmd = None
        python_path = self.selected
        python_path_id = self.python_list.get_active()

        if not python_path:
            ok = False
            self.textbuffer.set_text('Python exec not defined.')
            
        if not os.path.exists(python_path):
            ok = False
            self.textbuffer.set_text('Python exec not found: %s' % python_path)

        script_path = self.proj_path.get_text()
        script = self.script.get_text()  

        full_script_path = os.path.join(script_path,script)
        if not os.path.exists(full_script_path):
            ok = False
            self.textbuffer.set_text('Script not found: %s' % full_script_path)
        
        if ok:

            user_set = [python_path_id, script_path, script]
            save_user(user_set)

            self.textbuffer.set_text(self.pid)
            
            self.pid = 'PID: Unkown. Possibly ended quickly.'
            self.cmd = [python_path, full_script_path]

            thread = Worker(self.cmd, self.messages)
            thread.start()
            
            for process in psutil.process_iter():
                if process.cmdline() == self.cmd:
                    self.pid = 'PID = %s: %s' % (process.pid, process.cmdline())
                    self.textbuffer.set_text(self.pid)
                    break

    def messages(self,rvals):
        r_err = rvals[-1].decode()
        r_ok = rvals[0].decode()

        self.err = '\nResults:\n\n%s' % r_err
        self.msg = '\nResults:\n\n%s' % r_ok

        start_iter = self.textbuffer.get_start_iter()
        end_iter = self.textbuffer.get_end_iter()
        self.textbuffer.remove_all_tags(start_iter,end_iter)
        if r_err: 
            self.textbuffer.insert_with_tags(end_iter, self.err, self.tag_err)
        elif r_ok: 
            if 'exception' in r_ok.lower() or 'error' in r_ok.lower() and 'traceback' in r_ok.lower():
                self.textbuffer.insert_with_tags(end_iter, self.msg, self.tag_err)
            else:
                self.textbuffer.insert_with_tags(end_iter, self.msg, self.tag_msg)
        self.message_view.show()

if __name__ == '__main__':
    win = Executor()
    win.connect("destroy", Gtk.main_quit)
    win.show_all()
    Gtk.main()