RegExp Help to parse Smarty and Twig Files for translatable strings

[markus_blue_tomato]

Hello!

I am working on a pull request, to make Smarty and Twig templates translatable within ProcessWire.

I stuck a little bit at the RegExp patterns. Maybe some RegExp-Professional want's to help me? :-)

Should be possible in Smarty files:

{$this->__('text')}
{__('text')}
{_x('text')}
{_n('text')}

Should be possible in Twig Files:

{{ $this->__('text') }}
{{ __('text') }}
{{ _x('text') }}
{{ _n('text') }}

The whole patters are in the parseFile Function: https://github.com/processwire/processwire/blob/341342dc5b1c58012ae7cb26cffe2c57cd915552/wire/modules/LanguageSupport/LanguageParser.php#L120

/**
	 * Run regex's on file contents to locate all translation functions
	 *
	 */
	protected function parseFile($file) { 
		$matches = array(
			1 => array(), 	// $this->_('text'); 
			2 => array(),	// __('text', [textdomain]);
			3 => array(),	// _x('text', 'context', [textdomain]) or $this->_x('text', 'context'); 
			4 => array(),	// _n('singular', 'plural', $cnt, [textdomain]) or $this->_n(...); 
			);
		if(!is_file($file)) return $matches; 
		$data = file_get_contents($file); 
		// Find $this->_('text') style matches
		preg_match_all(	'/(>_)\(\s*' .				// $this->_( 
				'([\'"])(.+?)(?<!\\\\)\\2' . 		// "text"
				'\s*\)+(.*)$/m',					// and everything else
				$data, $matches[1]); 
		// Find __('text', textdomain) style matches
		preg_match_all(	'/([\s.=(]__|^__)\(\s*' . 		// __(
				'([\'"])(.+?)(?<!\\\\)\\2\s*' . 	// "text"
				'(?:,\s*[^)]+)?\)+(.*)$/m', 		// , textdomain (optional) and everything else
				$data, $matches[2]); 
		// Find _x('text', 'context', textdomain) or $this->_x('text', 'context') style matches
		preg_match_all(	'/([\s.=>(]_x|^_x)\(\s*' . 		// _x( or $this->_x(
				'([\'"])(.+?)(?<!\\\\)\\2\s*,\s*' . 	// "text", 
				'([\'"])(.+?)(?<!\\\\)\\4\s*' . 	// "context"
				'[^)]*\)+(.*)$/m',			// , textdomain (optional) and everything else 
				$data, $matches[3]); 
		// Find _n('singular text', 'plural text', $cnt, textdomain) or $this->_n(...) style matches
		preg_match_all(	'/([\s.=>(]_n|^_n)\(\s*' . 		// _n( or $this->_n(
				'([\'"])(.+?)(?<!\\\\)\\2\s*,\s*' . 	// "singular", 
				'([\'"])(.+?)(?<!\\\\)\\4\s*,\s*' . 	// "plural", 
				'.+?\)+(.*)$/m', 			// $count, optional textdomain, closing function parenthesis ) and rest of line
				$data, $matches[4]); 
		return $matches; 
	}

 

[Wanze]

Hi Markus,

Why do you want to parse it yourself? ? Twig and Smarty have built-in parsers and both offer to extend the parser with your own functions / filters / modifiers and so on. Twig even allows you to write custom expressions.

Here is an example how to integrate the "__('text') function into Twig, assuming you are using the TemplateEngineFactory in combination with the TemplateEngineTwig module.

        $this->addHookAfter('TemplateEngineTwig::initTwig', function (HookEvent $event) {
            /** @var \Twig_Environment $twig */
            $twig = $event->arguments('twig');
            
            $function = new \Twig_SimpleFunction('__', function ($key) {
                return __($key, "/site/templates/translations/strings.php");
            });
            
            $twig->addFunction($function);
        });

As you can see, I'm collecting all my translatable strings in a PHP file "strings.php". But you could also extend the twig function to optionally accept the text domain.

Cheers

[markus_blue_tomato]

5 hours ago, Wanze said:

 

As you can see, I'm collecting all my translatable strings in a PHP file "strings.php". But you could also extend the twig function to optionally accept the text domain.

I want to avoid to collect the strings in the PHP file and ProcessWire does not find the strings when they are in Smarty or Twig syntax.

[markus_blue_tomato]

@Wanze

I ended up in using your solution but to avoid writing the key everytime in two files per hand I created a node.js script which is executed in my build script of the whole project.

The scripts reads all translatable strings from my .tpl files which look like this {translate}Lorem Ipsum{/translate} and creates in every view-directory a translation.php file.

If anyone needs it here is the code of the node.js module (you have to install async and glob via npm..)

"use strict";

const glob = require('glob');
const async = require('async');
const fs = require('fs');

// find all directories
glob("site/templates/views/**/", null, (error, directories) => {
  // iterate in paralell over directories
  async.each(directories, function(directory, directoryCallback) {

    // find all template files in the current directory
    glob(`${directory}*.tpl`, null, (error, templateFiles) => {
      if(error) return directoryCallback(error);

      let translationKeys = [];
      // iterate in paralell over templateFiles
      async.each(templateFiles, function(templateFile, templateFileCallback) {
        fs.readFile(templateFile, (error, data) => {
          if(error) throw err;
          let tpl = data.toString('utf8');

          // find all keys
          let pattern = new RegExp('{translate\}(.+?)\{\/translate\}', 'gm');
          let result = tpl.match(pattern);
          if(result && result.length > 0) {
            // extract key without smarty block syntax
            // transform it to PHP syntax
            result = result.map(item => item.replace(pattern, '__("$1");'));
            // push all keys to the collector of all keys in the current directory
            translationKeys.push(...result);
          }
          templateFileCallback();

        });
      }, function(error) {
        if(error) console.log("Error in the templateFiles logic", error);

        // executed after alle keys are collected from the tpl files in the current directory
        if(translationKeys.length > 0) {

          // filter duplicates
          // sort alphabetical
          translationKeys = [ ...new Set(translationKeys) ].sort();

          // make the final PHP file
          let phpFile = [ '<?php namespace ProcessWire;', ...translationKeys ].join("\n");
          fs.writeFile(`${directory}translations.php`, phpFile, 'utf8', function(error) {
            if(error) throw err;
            directoryCallback();
          });
        } else {
          directoryCallback();
        }

      });

    });
  }, function(error) {
    if(error) console.log("Error in the directory logic", error);

    // all done exit the process
    process.exit();

  });
});